Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610// SPDX-License-Identifier: GPL-2.0/* * Copyright(C) 2005-2006, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner * * High-resolution kernel timers * * In contrast to the low-resolution timeout API, aka timer wheel, * hrtimers provide finer resolution and accuracy depending on system * configuration and capabilities. * * Started by: Thomas Gleixner and Ingo Molnar * * Credits: * Based on the original timer wheel code * * Help, testing, suggestions, bugfixes, improvements were * provided by: * * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel * et. al. */
#include <linux/cpu.h>#include <linux/export.h>#include <linux/percpu.h>#include <linux/hrtimer.h>#include <linux/notifier.h>#include <linux/syscalls.h>#include <linux/interrupt.h>#include <linux/tick.h>#include <linux/err.h>#include <linux/debugobjects.h>#include <linux/sched/signal.h>#include <linux/sched/sysctl.h>#include <linux/sched/rt.h>#include <linux/sched/deadline.h>#include <linux/sched/nohz.h>#include <linux/sched/debug.h>#include <linux/sched/isolation.h>#include <linux/timer.h>#include <linux/freezer.h>#include <linux/compat.h>
#include <linux/uaccess.h>
#include <trace/events/timer.h>
#include "tick-internal.h"
/* * Constants to set the queued state of the timer (INACTIVE, ENQUEUED) * * The callback state is kept separate in the CPU base because having it in * the timer would required touching the timer after the callback, which * makes it impossible to free the timer from the callback function. * * Therefore we track the callback state in: * * timer->base->cpu_base->running == timer * * On SMP it is possible to have a "callback function running and enqueued" * status. It happens for example when a posix timer expired and the callback * queued a signal. Between dropping the lock which protects the posix timer * and reacquiring the base lock of the hrtimer, another CPU can deliver the * signal and rearm the timer. * * All state transitions are protected by cpu_base->lock. */#define HRTIMER_STATE_INACTIVE false#define HRTIMER_STATE_ENQUEUED true
/* * The resolution of the clocks. The resolution value is returned in * the clock_getres() system call to give application programmers an * idea of the (in)accuracy of timers. Timer values are rounded up to * this resolution values. */#define HIGH_RES_NSEC 1
/* * Masks for selecting the soft and hard context timers from * cpu_base->active */#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
static void retrigger_next_event(void *arg);static ktime_t __hrtimer_cb_get_time(clockid_t clock_id);
/* * The timer bases: * * There are more clockids than hrtimer bases. Thus, we index * into the timer bases by the hrtimer_base_type enum. When trying * to reach a base using a clockid, hrtimer_clockid_to_base() * is used to convert from clockid to the proper hrtimer_base_type. */
#define BASE_INIT(idx, cid) \ [idx] = { .index = idx, .clockid = cid }
DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) ={ .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock), .clock_base = { BASE_INIT(HRTIMER_BASE_MONOTONIC, CLOCK_MONOTONIC), BASE_INIT(HRTIMER_BASE_REALTIME, CLOCK_REALTIME), BASE_INIT(HRTIMER_BASE_BOOTTIME, CLOCK_BOOTTIME), BASE_INIT(HRTIMER_BASE_TAI, CLOCK_TAI), BASE_INIT(HRTIMER_BASE_MONOTONIC_SOFT, CLOCK_MONOTONIC), BASE_INIT(HRTIMER_BASE_REALTIME_SOFT, CLOCK_REALTIME), BASE_INIT(HRTIMER_BASE_BOOTTIME_SOFT, CLOCK_BOOTTIME), BASE_INIT(HRTIMER_BASE_TAI_SOFT, CLOCK_TAI), }, .csd = CSD_INIT(retrigger_next_event, NULL)};
static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base){ if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) return true; else return likely(base->online);}
#ifdef CONFIG_HIGH_RES_TIMERSDEFINE_STATIC_KEY_FALSE(hrtimer_highres_enabled_key);
static void hrtimer_hres_workfn(struct work_struct *work){ static_branch_enable(&hrtimer_highres_enabled_key);}
static DECLARE_WORK(hrtimer_hres_work, hrtimer_hres_workfn);
static inline void hrtimer_schedule_hres_work(void){ if (!hrtimer_highres_enabled()) schedule_work(&hrtimer_hres_work);}#elsestatic inline void hrtimer_schedule_hres_work(void) { }#endif
/* * Functions and macros which are different for UP/SMP systems are kept in a * single place */#ifdef CONFIG_SMP/* * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base() * such that hrtimer_callback_running() can unconditionally dereference * timer->base->cpu_base */static struct hrtimer_cpu_base migration_cpu_base = { .clock_base = { [0] = { .cpu_base = &migration_cpu_base, .seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq, &migration_cpu_base.lock), }, },};
#define migration_base migration_cpu_base.clock_base[0]
/* * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock * means that all timers which are tied to this base via timer->base are * locked, and the base itself is locked too. * * So __run_timers/migrate_timers can safely modify all timers which could * be found on the lists/queues. * * When the timer's base is locked, and the timer removed from list, it is * possible to set timer->base = &migration_base and drop the lock: the timer * remains locked. */static struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) __acquires(&timer->base->lock){ for (;;) { struct hrtimer_clock_base *base = READ_ONCE(timer->base);
if (likely(base != &migration_base)) { raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); if (likely(base == timer->base)) return base; /* The timer has migrated to another CPU: */ raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); } cpu_relax(); }}
/* * Check if the elected target is suitable considering its next * event and the hotplug state of the current CPU. * * If the elected target is remote and its next event is after the timer * to queue, then a remote reprogram is necessary. However there is no * guarantee the IPI handling the operation would arrive in time to meet * the high resolution deadline. In this case the local CPU becomes a * preferred target, unless it is offline. * * High and low resolution modes are handled the same way for simplicity. * * Called with cpu_base->lock of target cpu held. */static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base, struct hrtimer_cpu_base *new_cpu_base, struct hrtimer_cpu_base *this_cpu_base){ ktime_t expires;
/* * The local CPU clockevent can be reprogrammed. Also get_target_base() * guarantees it is online. */ if (new_cpu_base == this_cpu_base) return true;
/* * The offline local CPU can't be the default target if the * next remote target event is after this timer. Keep the * elected new base. An IPI will be issued to reprogram * it as a last resort. */ if (!hrtimer_base_is_online(this_cpu_base)) return true;
expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
return expires >= new_base->cpu_base->expires_next;}
static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, bool pinned){ if (!hrtimer_base_is_online(base)) { int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER));
return &per_cpu(hrtimer_bases, cpu); }
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) if (static_branch_likely(&timers_migration_enabled) && !pinned) return &per_cpu(hrtimer_bases, get_nohz_timer_target());#endif return base;}
/* * We switch the timer base to a power-optimized selected CPU target, * if: * - NO_HZ_COMMON is enabled * - timer migration is enabled * - the timer callback is not running * - the timer is not the first expiring timer on the new target * * If one of the above requirements is not fulfilled we move the timer * to the current CPU or leave it on the previously assigned CPU if * the timer callback is currently running. */static inline struct hrtimer_clock_base *switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, bool pinned){ struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base; struct hrtimer_clock_base *new_base; int basenum = base->index;
this_cpu_base = this_cpu_ptr(&hrtimer_bases); new_cpu_base = get_target_base(this_cpu_base, pinned);again: new_base = &new_cpu_base->clock_base[basenum];
if (base != new_base) { /* * We are trying to move timer to new_base. However we can't * change timer's base while it is running, so we keep it on * the same CPU. No hassle vs. reprogramming the event source * in the high resolution case. The remote CPU will take care * of this when the timer function has completed. There is no * conflict as we hold the lock until the timer is enqueued. */ if (unlikely(hrtimer_callback_running(timer))) return base;
/* See the comment in lock_hrtimer_base() */ WRITE_ONCE(timer->base, &migration_base); raw_spin_unlock(&base->cpu_base->lock); raw_spin_lock(&new_base->cpu_base->lock);
if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) { raw_spin_unlock(&new_base->cpu_base->lock); raw_spin_lock(&base->cpu_base->lock); new_cpu_base = this_cpu_base; WRITE_ONCE(timer->base, base); goto again; } WRITE_ONCE(timer->base, new_base); } else { if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) { new_cpu_base = this_cpu_base; goto again; } } return new_base;}
#else /* CONFIG_SMP */
static inline struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) __acquires(&timer->base->cpu_base->lock){ struct hrtimer_clock_base *base = timer->base;
raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); return base;}
# define switch_hrtimer_base(t, b, p) (b)
#endif /* !CONFIG_SMP */
/* * Functions for the union type storage format of ktime_t which are * too large for inlining: */#if BITS_PER_LONG < 64/* * Divide a ktime value by a nanosecond value */s64 __ktime_divns(const ktime_t kt, s64 div){ int sft = 0; s64 dclc; u64 tmp;
dclc = ktime_to_ns(kt); tmp = dclc < 0 ? -dclc : dclc;
/* Make sure the divisor is less than 2^32: */ while (div >> 32) { sft++; div >>= 1; } tmp >>= sft; do_div(tmp, (u32) div); return dclc < 0 ? -tmp : tmp;}EXPORT_SYMBOL_GPL(__ktime_divns);#endif /* BITS_PER_LONG < 64 */
/* * Add two ktime values and do a safety check for overflow: */ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs){ ktime_t res = ktime_add_unsafe(lhs, rhs);
/* * We use KTIME_SEC_MAX here, the maximum timeout which we can * return to user space in a timespec: */ if (res < 0 || res < lhs || res < rhs) res = ktime_set(KTIME_SEC_MAX, 0);
return res;}
EXPORT_SYMBOL_GPL(ktime_add_safe);
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
static const struct debug_obj_descr hrtimer_debug_descr;
static void *hrtimer_debug_hint(void *addr){ return ACCESS_PRIVATE((struct hrtimer *)addr, function);}
/* * fixup_init is called when: * - an active object is initialized */static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state){ struct hrtimer *timer = addr;
switch (state) { case ODEBUG_STATE_ACTIVE: hrtimer_cancel(timer); debug_object_init(timer, &hrtimer_debug_descr); return true; default: return false; }}
/* * fixup_activate is called when: * - an active object is activated * - an unknown non-static object is activated */static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state){ switch (state) { case ODEBUG_STATE_ACTIVE: WARN_ON(1); fallthrough; default: return false; }}
/* * fixup_free is called when: * - an active object is freed */static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state){ struct hrtimer *timer = addr;
switch (state) { case ODEBUG_STATE_ACTIVE: hrtimer_cancel(timer); debug_object_free(timer, &hrtimer_debug_descr); return true; default: return false; }}
/* Stub timer callback for improperly used timers. */static enum hrtimer_restart stub_timer(struct hrtimer *unused){ WARN_ON_ONCE(1); return HRTIMER_NORESTART;}
/* * hrtimer_fixup_assert_init is called when: * - an untracked/uninit-ed object is found */static bool hrtimer_fixup_assert_init(void *addr, enum debug_obj_state state){ struct hrtimer *timer = addr;
switch (state) { case ODEBUG_STATE_NOTAVAILABLE: hrtimer_setup(timer, stub_timer, CLOCK_MONOTONIC, 0); return true; default: return false; }}
static const struct debug_obj_descr hrtimer_debug_descr = { .name = "hrtimer", .debug_hint = hrtimer_debug_hint, .fixup_init = hrtimer_fixup_init, .fixup_activate = hrtimer_fixup_activate, .fixup_free = hrtimer_fixup_free, .fixup_assert_init = hrtimer_fixup_assert_init,};
static inline void debug_hrtimer_init(struct hrtimer *timer){ debug_object_init(timer, &hrtimer_debug_descr);}
static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer){ debug_object_init_on_stack(timer, &hrtimer_debug_descr);}
static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode){ debug_object_activate(timer, &hrtimer_debug_descr);}
static inline void debug_hrtimer_deactivate(struct hrtimer *timer){ debug_object_deactivate(timer, &hrtimer_debug_descr);}
static inline void debug_hrtimer_assert_init(struct hrtimer *timer){ debug_object_assert_init(timer, &hrtimer_debug_descr);}
void destroy_hrtimer_on_stack(struct hrtimer *timer){ debug_object_free(timer, &hrtimer_debug_descr);}EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
#else
static inline void debug_hrtimer_init(struct hrtimer *timer) { }static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode) { }static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }static inline void debug_hrtimer_assert_init(struct hrtimer *timer) { }#endif
static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode){ debug_hrtimer_init(timer); trace_hrtimer_setup(timer, clockid, mode);}
static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode){ debug_hrtimer_init_on_stack(timer); trace_hrtimer_setup(timer, clockid, mode);}
static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed){ debug_hrtimer_activate(timer, mode); trace_hrtimer_start(timer, mode, was_armed);}
#define for_each_active_base(base, cpu_base, active) \ for (unsigned int idx = ffs(active); idx--; idx = ffs((active))) \ for (bool done = false; !done; active &= ~(1U << idx)) \ for (base = &cpu_base->clock_base[idx]; !done; done = true)
#define hrtimer_from_timerqueue_node(_n) container_of_const(_n, struct hrtimer, node)
#if defined(CONFIG_NO_HZ_COMMON)/* * Same as hrtimer_bases_next_event() below, but skips the excluded timer and * does not update cpu_base->next_timer/expires. */static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base, const struct hrtimer *exclude, unsigned int active, ktime_t expires_next){ struct hrtimer_clock_base *base; ktime_t expires;
lockdep_assert_held(&cpu_base->lock);
for_each_active_base(base, cpu_base, active) { expires = ktime_sub(base->expires_next, base->offset); if (expires >= expires_next) continue;
/* * If the excluded timer is the first on this base evaluate the * next timer. */ struct timerqueue_linked_node *node = timerqueue_linked_first(&base->active);
if (unlikely(&exclude->node == node)) { node = timerqueue_linked_next(node); if (!node) continue; expires = ktime_sub(node->expires, base->offset); if (expires >= expires_next) continue; } expires_next = expires; } /* If base->offset changed, the result might be negative */ return max(expires_next, 0);}#endif
static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base){ struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
return hrtimer_from_timerqueue_node(next);}
/* Find the base with the earliest expiry */static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active, ktime_t *expires_next, struct hrtimer **next_timer){ struct hrtimer_clock_base *base; ktime_t expires;
for_each_active_base(base, cpu_base, active) { expires = ktime_sub(base->expires_next, base->offset); if (expires < *expires_next) { *expires_next = expires; *next_timer = clock_base_next_timer(base); } }}
/* * Recomputes cpu_base::*next_timer and returns the earliest expires_next * but does not set cpu_base::*expires_next, that is done by * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating * cpu_base::*expires_next right away, reprogramming logic would no longer * work. * * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases, * those timers will get run whenever the softirq gets handled, at the end of * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases. * * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases. * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD. * * @active_mask must be one of: * - HRTIMER_ACTIVE_ALL, * - HRTIMER_ACTIVE_SOFT, or * - HRTIMER_ACTIVE_HARD. */static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask){ struct hrtimer *next_timer = NULL; ktime_t expires_next = KTIME_MAX; unsigned int active;
lockdep_assert_held(&cpu_base->lock);
if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; if (active) hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer); cpu_base->softirq_next_timer = next_timer; }
if (active_mask & HRTIMER_ACTIVE_HARD) { active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; if (active) hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer); cpu_base->next_timer = next_timer; } return max(expires_next, 0);}
static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base){ ktime_t expires_next, soft = KTIME_MAX;
/* * If the soft interrupt has already been activated, ignore the * soft bases. They will be handled in the already raised soft * interrupt. */ if (!cpu_base->softirq_activated) { soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT); /* * Update the soft expiry time. clock_settime() might have * affected it. */ cpu_base->softirq_expires_next = soft; }
expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD); /* * If a softirq timer is expiring first, update cpu_base->next_timer * and program the hardware with the soft expiry time. */ if (expires_next > soft) { cpu_base->next_timer = cpu_base->softirq_next_timer; expires_next = soft; }
return expires_next;}
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base){ ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, offs_real, offs_boot, offs_tai);
base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
return now;}
/* * Is the high resolution mode active in the CPU base. This cannot use the * static key as the CPUs are switched to high resolution mode * asynchronously. */static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base){ return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ? cpu_base->hres_active : 0;}
static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred){ trace_hrtimer_rearm(expires_next, deferred); tick_program_event(expires_next, 1);}
static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer, ktime_t expires_next){ cpu_base->expires_next = expires_next;
/* * If hres is not active, hardware does not have to be * reprogrammed yet. * * If a hang was detected in the last timer interrupt then we * leave the hang delay active in the hardware. We want the * system to make progress. That also prevents the following * scenario: * T1 expires 50ms from now * T2 expires 5s from now * * T1 is removed, so this code is called and would reprogram * the hardware to 5s from now. Any hrtimer_start after that * will not reprogram the hardware due to hang_detected being * set. So we'd effectively block all timers until the T2 event * fires. */ if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) return;
hrtimer_rearm_event(expires_next, false);}
/* Reprogram the event source with a evaluation of all clock bases */static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal){ ktime_t expires_next = hrtimer_update_next_event(cpu_base);
if (skip_equal && expires_next == cpu_base->expires_next) return;
__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);}
/* High resolution timer related functions */#ifdef CONFIG_HIGH_RES_TIMERS
/* High resolution timer enabled ? */static bool hrtimer_hres_enabled __read_mostly = true;unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;EXPORT_SYMBOL_GPL(hrtimer_resolution);
/* Enable / Disable high resolution mode */static int __init setup_hrtimer_hres(char *str){ return (kstrtobool(str, &hrtimer_hres_enabled) == 0);}__setup("highres=", setup_hrtimer_hres);
/* hrtimer_high_res_enabled - query, if the highres mode is enabled */static inline bool hrtimer_is_hres_enabled(void){ return hrtimer_hres_enabled;}
/* Switch to high resolution mode */static void hrtimer_switch_to_hres(void){ struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
if (tick_init_highres()) { pr_warn("Could not switch to high resolution mode on CPU %u\n", base->cpu); return; } base->hres_active = true; hrtimer_resolution = HIGH_RES_NSEC;
tick_setup_sched_timer(true); /* "Retrigger" the interrupt to get things going */ retrigger_next_event(NULL); hrtimer_schedule_hres_work();}
#else
static inline bool hrtimer_is_hres_enabled(void) { return 0; }static inline void hrtimer_switch_to_hres(void) { }
#endif /* CONFIG_HIGH_RES_TIMERS */
/* * Retrigger next event is called after clock was set with interrupts * disabled through an SMP function call or directly from low level * resume code. * * This is only invoked when: * - CONFIG_HIGH_RES_TIMERS is enabled. * - CONFIG_NO_HZ_COMMON is enabled * * For the other cases this function is empty and because the call sites * are optimized out it vanishes as well, i.e. no need for lots of * #ifdeffery. */static void retrigger_next_event(void *arg){ struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
/* * When high resolution mode or nohz is active, then the offsets of * CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the * next tick will take care of that. * * If high resolution mode is active then the next expiring timer * must be reevaluated and the clock event device reprogrammed if * necessary. * * In the NOHZ case the update of the offset and the reevaluation * of the next expiring timer is enough. The return from the SMP * function call will take care of the reprogramming in case the * CPU was in a NOHZ idle sleep. * * In periodic low resolution mode, the next softirq expiration * must also be updated. */ guard(raw_spinlock)(&base->lock); hrtimer_update_base(base); if (hrtimer_hres_active(base)) hrtimer_force_reprogram(base, /* skip_equal */ false); else hrtimer_update_next_event(base);}
/* * When a timer is enqueued and expires earlier than the already enqueued * timers, we have to check, whether it expires earlier than the timer for * which the clock event device was armed. * * Called with interrupts disabled and base->cpu_base.lock held */static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); struct hrtimer_clock_base *base = timer->base; ktime_t expires = hrtimer_get_expires(timer);
WARN_ON_ONCE(expires < 0);
expires = ktime_sub(expires, base->offset); /* * CLOCK_REALTIME timer might be requested with an absolute * expiry time which is less than base->offset. Set it to 0. */ if (expires < 0) expires = 0;
if (timer->is_soft) { /* * soft hrtimer could be started on a remote CPU. In this * case softirq_expires_next needs to be updated on the * remote CPU. The soft hrtimer will not expire before the * first hard hrtimer on the remote CPU - * hrtimer_check_target() prevents this case. */ struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
if (timer_cpu_base->softirq_activated) return;
if (!ktime_before(expires, timer_cpu_base->softirq_expires_next)) return;
timer_cpu_base->softirq_next_timer = timer; timer_cpu_base->softirq_expires_next = expires;
if (!ktime_before(expires, timer_cpu_base->expires_next) || !reprogram) return; }
/* * If the timer is not on the current cpu, we cannot reprogram * the other cpus clock event device. */ if (base->cpu_base != cpu_base) return;
if (expires >= cpu_base->expires_next) return;
/* If a deferred rearm is pending skip reprogramming the device */ if (cpu_base->deferred_rearm) return;
cpu_base->next_timer = timer;
__hrtimer_reprogram(cpu_base, timer, expires);}
static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active){ struct hrtimer_clock_base *base; unsigned int seq; ktime_t expires;
/* * Update the base offsets unconditionally so the following * checks whether the SMP function call is required works. * * The update is safe even when the remote CPU is in the hrtimer * interrupt or the hrtimer soft interrupt and expiring affected * bases. Either it will see the update before handling a base or * it will see it when it finishes the processing and reevaluates * the next expiring timer. */ seq = cpu_base->clock_was_set_seq; hrtimer_update_base(cpu_base);
/* * If the sequence did not change over the update then the * remote CPU already handled it. */ if (seq == cpu_base->clock_was_set_seq) return false;
/* If a deferred rearm is pending the remote CPU will take care of it */ if (cpu_base->deferred_rearm) { cpu_base->deferred_needs_update = true; return false; }
/* * Walk the affected clock bases and check whether the first expiring * timer in a clock base is moving ahead of the first expiring timer of * @cpu_base. If so, the IPI must be invoked because per CPU clock * event devices cannot be remotely reprogrammed. */ active &= cpu_base->active_bases;
for_each_active_base(base, cpu_base, active) { struct timerqueue_linked_node *next;
next = timerqueue_linked_first(&base->active); expires = ktime_sub(next->expires, base->offset); if (expires < cpu_base->expires_next) return true;
/* Extra check for softirq clock bases */ if (base->index < HRTIMER_BASE_MONOTONIC_SOFT) continue; if (cpu_base->softirq_activated) continue; if (expires < cpu_base->softirq_expires_next) return true; } return false;}
/* * Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and * CLOCK_BOOTTIME (for late sleep time injection). * * This requires to update the offsets for these clocks * vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this * also requires to eventually reprogram the per CPU clock event devices * when the change moves an affected timer ahead of the first expiring * timer on that CPU. Obviously remote per CPU clock event devices cannot * be reprogrammed. The other reason why an IPI has to be sent is when the * system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets * in the tick, which obviously might be stopped, so this has to bring out * the remote CPU which might sleep in idle to get this sorted. */void clock_was_set(unsigned int bases){ cpumask_var_t mask;
if (!hrtimer_highres_enabled() && !tick_nohz_is_active()) goto out_timerfd;
if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { on_each_cpu(retrigger_next_event, NULL, 1); goto out_timerfd; }
/* Avoid interrupting CPUs if possible */ scoped_guard(cpus_read_lock) { int cpu;
for_each_online_cpu(cpu) { struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
guard(raw_spinlock_irqsave)(&cpu_base->lock); if (update_needs_ipi(cpu_base, bases)) cpumask_set_cpu(cpu, mask); } scoped_guard(preempt) smp_call_function_many(mask, retrigger_next_event, NULL, 1); } free_cpumask_var(mask);
out_timerfd: timerfd_clock_was_set();}
static void clock_was_set_work(struct work_struct *work){ clock_was_set(CLOCK_SET_WALL);}
static DECLARE_WORK(hrtimer_work, clock_was_set_work);
/* * Called from timekeeping code to reprogram the hrtimer interrupt device * on all cpus and to notify timerfd. */void clock_was_set_delayed(void){ schedule_work(&hrtimer_work);}
/* * Called during resume either directly from via timekeeping_resume() * or in the case of s2idle from tick_unfreeze() to ensure that the * hrtimers are up to date. */void hrtimers_resume_local(void){ lockdep_assert_irqs_disabled(); /* Retrigger on the local CPU */ retrigger_next_event(NULL);}
/* Counterpart to lock_hrtimer_base above */static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) __releases(&timer->base->cpu_base->lock){ raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);}
/** * hrtimer_forward() - forward the timer expiry * @timer: hrtimer to forward * @now: forward past this time * @interval: the interval to forward * * Forward the timer expiry so it will expire in the future. * * .. note:: * This only updates the timer expiry value and does not requeue the timer. * * There is also a variant of this function: hrtimer_forward_now(). * * Context: Can be safely called from the callback function of @timer. If called * from other contexts @timer must neither be enqueued nor running the * callback and the caller needs to take care of serialization. * * Return: The number of overruns are returned. */u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval){ ktime_t delta; u64 orun = 1;
delta = ktime_sub(now, hrtimer_get_expires(timer));
if (delta < 0) return 0;
if (WARN_ON(timer->is_queued)) return 0;
if (interval < hrtimer_resolution) interval = hrtimer_resolution;
if (unlikely(delta >= interval)) { s64 incr = ktime_to_ns(interval);
orun = ktime_divns(delta, incr); hrtimer_add_expires_ns(timer, incr * orun); if (hrtimer_get_expires(timer) > now) return orun; /* * This (and the ktime_add() below) is the * correction for exact: */ orun++; } hrtimer_add_expires(timer, interval);
return orun;}EXPORT_SYMBOL_GPL(hrtimer_forward);
/* * enqueue_hrtimer - internal function to (re)start a timer * * The timer is inserted in expiry order. Insertion into the * red black tree is O(log(n)). * * Returns true when the new timer is the leftmost timer in the tree. */static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, enum hrtimer_mode mode, bool was_armed){ lockdep_assert_held(&base->cpu_base->lock);
debug_activate(timer, mode, was_armed); WARN_ON_ONCE(!base->cpu_base->online);
base->cpu_base->active_bases |= 1 << base->index;
/* Pairs with the lockless read in hrtimer_is_queued() */ WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
if (!timerqueue_linked_add(&base->active, &timer->node)) return false;
base->expires_next = hrtimer_get_expires(timer); return true;}
static inline void base_update_next_timer(struct hrtimer_clock_base *base){ struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
base->expires_next = next ? next->expires : KTIME_MAX;}
/* * __remove_hrtimer - internal function to remove a timer * * High resolution timer mode reprograms the clock event device when the * timer is the one which expires next. The caller can disable this by setting * reprogram to zero. This is useful, when the context does a reprogramming * anyway (e.g. timer interrupt) */static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool newstate, bool reprogram){ struct hrtimer_cpu_base *cpu_base = base->cpu_base; bool was_first;
lockdep_assert_held(&cpu_base->lock);
if (!timer->is_queued) return;
/* Pairs with the lockless read in hrtimer_is_queued() */ WRITE_ONCE(timer->is_queued, newstate);
was_first = !timerqueue_linked_prev(&timer->node);
if (!timerqueue_linked_del(&base->active, &timer->node)) cpu_base->active_bases &= ~(1 << base->index);
/* Nothing to update if this was not the first timer in the base */ if (!was_first) return;
base_update_next_timer(base);
/* * If reprogram is false don't update cpu_base->next_timer and do not * touch the clock event device. * * This happens when removing the first timer on a remote CPU, which * will be handled by the remote CPU's interrupt. It also happens when * a local timer is removed to be immediately restarted. That's handled * at the call site. */ if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy) return;
if (cpu_base->deferred_rearm) cpu_base->deferred_needs_update = true; else hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);}
static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool newstate){ lockdep_assert_held(&base->cpu_base->lock);
if (timer->is_queued) { bool reprogram;
debug_hrtimer_deactivate(timer);
/* * Remove the timer and force reprogramming when high * resolution mode is active and the timer is on the current * CPU. If we remove a timer on another CPU, reprogramming is * skipped. The interrupt event on this CPU is fired and * reprogramming happens in the interrupt handler. This is a * rare case and less expensive than a smp call. */ reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
__remove_hrtimer(timer, base, newstate, reprogram); return true; } return false;}
/* * Update in place has to retrieve the expiry times of the neighbour nodes * if they exist. That is cache line neutral because the dequeue/enqueue * operation is going to need the same cache lines. But there is a big win * when the dequeue/enqueue can be avoided because the RB tree does not * have to be rebalanced twice. */static inline boolhrtimer_can_update_in_place(struct hrtimer *timer, struct hrtimer_clock_base *base, ktime_t expires){ struct timerqueue_linked_node *next = timerqueue_linked_next(&timer->node); struct timerqueue_linked_node *prev = timerqueue_linked_prev(&timer->node);
/* If the new expiry goes behind the next timer, requeue is required */ if (next && expires > next->expires) return false;
/* If this is the first timer, update in place */ if (!prev) return true;
/* Update in place when it does not go ahead of the previous one */ return expires >= prev->expires;}
static inline boolremove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base, const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns){ bool was_first = false;
/* Remove it from the timer queue if active */ if (timer->is_queued) { was_first = !timerqueue_linked_prev(&timer->node);
/* Try to update in place to avoid the de/enqueue dance */ if (hrtimer_can_update_in_place(timer, base, expires)) { hrtimer_set_expires_range_ns(timer, expires, delta_ns); trace_hrtimer_start(timer, mode, true); if (was_first) base->expires_next = expires; return was_first; }
debug_hrtimer_deactivate(timer); timerqueue_linked_del(&base->active, &timer->node); }
/* Set the new expiry time */ hrtimer_set_expires_range_ns(timer, expires, delta_ns);
debug_activate(timer, mode, timer->is_queued); base->cpu_base->active_bases |= 1 << base->index;
/* Pairs with the lockless read in hrtimer_is_queued() */ WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
/* If it's the first expiring timer now or again, update base */ if (timerqueue_linked_add(&base->active, &timer->node)) { base->expires_next = expires; return true; }
if (was_first) base_update_next_timer(base);
return false;}
static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode){#ifdef CONFIG_TIME_LOW_RES /* * CONFIG_TIME_LOW_RES indicates that the system has no way to return * granular time values. For relative timers we add hrtimer_resolution * (i.e. one jiffy) to prevent short timeouts. */ timer->is_rel = mode & HRTIMER_MODE_REL; if (timer->is_rel) tim = ktime_add_safe(tim, hrtimer_resolution);#endif return tim;}
static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram){ ktime_t expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
/* * Reprogramming needs to be triggered, even if the next soft * hrtimer expires at the same time as the next hard * hrtimer. cpu_base->softirq_expires_next needs to be updated! */ if (expires == KTIME_MAX) return;
/* * cpu_base->next_timer is recomputed by __hrtimer_get_next_event() * cpu_base->expires_next is only set by hrtimer_reprogram() */ hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);}
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned){ if (static_branch_likely(&timers_migration_enabled)) { /* * If it is local and the first expiring timer keep it on the local * CPU to optimize reprogramming of the clockevent device. Also * avoid switch_hrtimer_base() overhead when local and pinned. */ if (!is_local) return false; if (is_first || is_pinned) return true;
/* Honour the NOHZ full restrictions */ if (!housekeeping_cpu(smp_processor_id(), HK_TYPE_KERNEL_NOISE)) return false;
/* * If the tick is not stopped or need_resched() is set, then * there is no point in moving the timer somewhere else. */ return !tick_nohz_tick_stopped() || need_resched(); } return is_local;}#elsestatic __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned){ return is_local;}#endif
static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool is_first, bool is_pinned){ /* If the timer is running the callback it has to stay on its CPU base. */ if (unlikely(timer->base->running == timer)) return true;
return hrtimer_prefer_local(is_local, is_first, is_pinned);}
enum { HRTIMER_REPROGRAM_NONE, HRTIMER_REPROGRAM, HRTIMER_REPROGRAM_FORCE,};
static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode, struct hrtimer_clock_base *base){ struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases); bool is_pinned, first, was_first, keep_base = false; struct hrtimer_cpu_base *cpu_base = base->cpu_base;
was_first = cpu_base->next_timer == timer; is_pinned = !!(mode & HRTIMER_MODE_PINNED);
/* * Don't keep it local if this enqueue happens on a unplugged CPU * after hrtimer_cpu_dying() has been invoked. */ if (likely(this_cpu_base->online)) { bool is_local = cpu_base == this_cpu_base;
keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned); }
/* Calculate absolute expiry time for relative timers */ if (mode & HRTIMER_MODE_REL) tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid)); /* Compensate for low resolution granularity */ tim = hrtimer_update_lowres(timer, tim, mode);
/* * Remove an active timer from the queue. In case it is not queued * on the current CPU, make sure that remove_hrtimer() updates the * remote data correctly. * * If it's on the current CPU and the first expiring timer, then * skip reprogramming, keep the timer local and enforce * reprogramming later if it was the first expiring timer. This * avoids programming the underlying clock event twice (once at * removal and once after enqueue). * * @keep_base is also true if the timer callback is running on a * remote CPU and for local pinned timers. */ if (likely(keep_base)) { first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns); } else { /* Keep the ENQUEUED state in case it is queued */ bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED);
hrtimer_set_expires_range_ns(timer, tim, delta_ns);
/* Switch the timer base, if necessary: */ base = switch_hrtimer_base(timer, base, is_pinned); cpu_base = base->cpu_base;
first = enqueue_hrtimer(timer, base, mode, was_armed); }
/* If a deferred rearm is pending skip reprogramming the device */ if (cpu_base->deferred_rearm) { cpu_base->deferred_needs_update = true; return HRTIMER_REPROGRAM_NONE; }
if (!was_first || cpu_base != this_cpu_base) { /* * If the current CPU base is online, then the timer is never * queued on a remote CPU if it would be the first expiring * timer there unless the timer callback is currently executed * on the remote CPU. In the latter case the remote CPU will * re-evaluate the first expiring timer after completing the * callbacks. */ if (likely(hrtimer_base_is_online(this_cpu_base))) return first ? HRTIMER_REPROGRAM : HRTIMER_REPROGRAM_NONE;
/* * Timer was enqueued remote because the current base is * already offline. If the timer is the first to expire, * kick the remote CPU to reprogram the clock event. */ if (first) smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd); return HRTIMER_REPROGRAM_NONE; }
/* * Special case for the HRTICK timer. It is frequently rearmed and most * of the time moves the expiry into the future. That's expensive in * virtual machines and it's better to take the pointless already armed * interrupt than reprogramming the hardware on every context switch. * * If the new expiry is before the armed time, then reprogramming is * required. */ if (timer->is_lazy) { if (cpu_base->expires_next <= hrtimer_get_expires(timer)) return HRTIMER_REPROGRAM_NONE; }
/* * Timer was the first expiring timer and forced to stay on the * current CPU to avoid reprogramming on removal and enqueue. Force * reprogram the hardware by evaluating the new first expiring * timer. */ return HRTIMER_REPROGRAM_FORCE;}
static int hrtimer_start_range_ns_common(struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode, struct hrtimer_clock_base *base){ /* * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard * expiry mode because unmarked timers are moved to softirq expiry. */ if (!IS_ENABLED(CONFIG_PREEMPT_RT)) WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); else WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, base);}
/** * hrtimer_start_range_ns - (re)start an hrtimer * @timer: the timer to be added * @tim: expiry time * @delta_ns: "slack" range for the timer * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED); * softirq based mode is considered for debug purpose only! */void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode){ struct hrtimer_clock_base *base; unsigned long flags;
debug_hrtimer_assert_init(timer);
base = lock_hrtimer_base(timer, &flags);
switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) { case HRTIMER_REPROGRAM: hrtimer_reprogram(timer, true); break; case HRTIMER_REPROGRAM_FORCE: hrtimer_force_reprogram(timer->base->cpu_base, 1); break; case HRTIMER_REPROGRAM_NONE: break; }
unlock_hrtimer_base(timer, &flags);}EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
static inline bool hrtimer_check_user_timer(struct hrtimer *timer){ struct hrtimer_cpu_base *cpu_base = timer->base->cpu_base; ktime_t expires;
/* * This uses soft expires because that's the user provided * expiry time, while expires can be further in the past * due to a slack value added to the user expiry time. */ expires = hrtimer_get_softexpires(timer);
/* Convert to monotonic */ expires = ktime_sub(expires, timer->base->offset);
/* * Check whether this timer will end up as the first expiring timer in * the CPU base. If not, no further checks required as it's then * guaranteed to expire in the future. */ if (expires >= cpu_base->expires_next) return true;
/* Validate that the expiry time is in the future. */ if (expires > ktime_get()) return true;
debug_hrtimer_deactivate(timer); __remove_hrtimer(timer, timer->base, HRTIMER_STATE_INACTIVE, false); trace_hrtimer_start_expired(timer); return false;}
/** * hrtimer_start_range_ns_user - (re)start an user controlled hrtimer * @timer: the timer to be added * @tim: expiry time * @delta_ns: "slack" range for the timer * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED); * softirq based mode is considered for debug purpose only! * * Returns: True when the timer was queued, false if it was already expired * * This function cannot invoke the timer callback for expired timers as it might * be called under a lock which the timer callback needs to acquire. So the * caller has to handle that case. */bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode){ struct hrtimer_clock_base *base; unsigned long flags; bool ret = true;
debug_hrtimer_assert_init(timer);
base = lock_hrtimer_base(timer, &flags);
switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) { case HRTIMER_REPROGRAM: ret = hrtimer_check_user_timer(timer); if (ret) hrtimer_reprogram(timer, true); break; case HRTIMER_REPROGRAM_FORCE: ret = hrtimer_check_user_timer(timer); /* * The base must always be reevaluated, independent of the * result above because the timer was the first pending timer. */ hrtimer_force_reprogram(timer->base->cpu_base, 1); break; case HRTIMER_REPROGRAM_NONE: break; }
unlock_hrtimer_base(timer, &flags); return ret;}EXPORT_SYMBOL_GPL(hrtimer_start_range_ns_user);
/** * hrtimer_try_to_cancel - try to deactivate a timer * @timer: hrtimer to stop * * Returns: * * * 0 when the timer was not active * * 1 when the timer was active * * -1 when the timer is currently executing the callback function and * cannot be stopped */int hrtimer_try_to_cancel(struct hrtimer *timer){ struct hrtimer_clock_base *base; unsigned long flags; int ret = -1;
/* * Check lockless first. If the timer is not active (neither * enqueued nor running the callback, nothing to do here. The * base lock does not serialize against a concurrent enqueue, * so we can avoid taking it. */ if (!hrtimer_active(timer)) return 0;
base = lock_hrtimer_base(timer, &flags);
if (!hrtimer_callback_running(timer)) { ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE); if (ret) trace_hrtimer_cancel(timer); }
unlock_hrtimer_base(timer, &flags);
return ret;
}EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
#ifdef CONFIG_PREEMPT_RTstatic void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base){ spin_lock_init(&base->softirq_expiry_lock);}
static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) __acquires(&base->softirq_expiry_lock){ spin_lock(&base->softirq_expiry_lock);}
static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) __releases(&base->softirq_expiry_lock){ spin_unlock(&base->softirq_expiry_lock);}
/* * The counterpart to hrtimer_cancel_wait_running(). * * If there is a waiter for cpu_base->expiry_lock, then it was waiting for * the timer callback to finish. Drop expiry_lock and reacquire it. That * allows the waiter to acquire the lock and make progress. */static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, unsigned long flags){ if (atomic_read(&cpu_base->timer_waiters)) { raw_spin_unlock_irqrestore(&cpu_base->lock, flags); spin_unlock(&cpu_base->softirq_expiry_lock); spin_lock(&cpu_base->softirq_expiry_lock); raw_spin_lock_irq(&cpu_base->lock); }}
#ifdef CONFIG_SMPstatic __always_inline bool is_migration_base(struct hrtimer_clock_base *base){ return base == &migration_base;}#elsestatic __always_inline bool is_migration_base(struct hrtimer_clock_base *base){ return false;}#endif
/* * This function is called on PREEMPT_RT kernels when the fast path * deletion of a timer failed because the timer callback function was * running. * * This prevents priority inversion: if the soft irq thread is preempted * in the middle of a timer callback, then calling hrtimer_cancel() can * lead to two issues: * * - If the caller is on a remote CPU then it has to spin wait for the timer * handler to complete. This can result in unbound priority inversion. * * - If the caller originates from the task which preempted the timer * handler on the same CPU, then spin waiting for the timer handler to * complete is never going to end. */void hrtimer_cancel_wait_running(const struct hrtimer *timer){ /* Lockless read. Prevent the compiler from reloading it below */ struct hrtimer_clock_base *base = READ_ONCE(timer->base);
/* * Just relax if the timer expires in hard interrupt context or if * it is currently on the migration base. */ if (!timer->is_soft || is_migration_base(base)) { cpu_relax(); return; }
/* * Mark the base as contended and grab the expiry lock, which is * held by the softirq across the timer callback. Drop the lock * immediately so the softirq can expire the next timer. In theory * the timer could already be running again, but that's more than * unlikely and just causes another wait loop. */ atomic_inc(&base->cpu_base->timer_waiters); spin_lock_bh(&base->cpu_base->softirq_expiry_lock); atomic_dec(&base->cpu_base->timer_waiters); spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);}#elsestatic inline void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }static inline void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }static inline void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base, unsigned long fl) { }#endif
/** * hrtimer_cancel - cancel a timer and wait for the handler to finish. * @timer: the timer to be cancelled * * Returns: * 0 when the timer was not active * 1 when the timer was active */int hrtimer_cancel(struct hrtimer *timer){ int ret;
do { ret = hrtimer_try_to_cancel(timer);
if (ret < 0) hrtimer_cancel_wait_running(timer); } while (ret < 0); return ret;}EXPORT_SYMBOL_GPL(hrtimer_cancel);
/** * __hrtimer_get_remaining - get remaining time for the timer * @timer: the timer to read * @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y */ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust){ unsigned long flags; ktime_t rem;
lock_hrtimer_base(timer, &flags); if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust) rem = hrtimer_expires_remaining_adjusted(timer); else rem = hrtimer_expires_remaining(timer); unlock_hrtimer_base(timer, &flags);
return rem;}EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
#ifdef CONFIG_NO_HZ_COMMON/** * hrtimer_get_next_event - get the time until next expiry event * * Returns the next expiry time or KTIME_MAX if no timer is pending. */ktime_t hrtimer_get_next_event(void){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); ktime_t expires = KTIME_MAX;
guard(raw_spinlock_irqsave)(&cpu_base->lock); if (!hrtimer_hres_active(cpu_base)) expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
return expires;}
/** * hrtimer_next_event_without - time until next expiry event w/o one timer * @exclude: timer to exclude * * Returns the next expiry time over all timers except for the @exclude one or * KTIME_MAX if none of them is pending. */ktime_t hrtimer_next_event_without(const struct hrtimer *exclude){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); ktime_t expires = KTIME_MAX; unsigned int active;
guard(raw_spinlock_irqsave)(&cpu_base->lock); if (!hrtimer_hres_active(cpu_base)) return expires;
active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; if (active && !cpu_base->softirq_activated) expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX);
active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; if (!active) return expires; return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires);}#endif
static inline int hrtimer_clockid_to_base(clockid_t clock_id){ switch (clock_id) { case CLOCK_MONOTONIC: return HRTIMER_BASE_MONOTONIC; case CLOCK_REALTIME: return HRTIMER_BASE_REALTIME; case CLOCK_BOOTTIME: return HRTIMER_BASE_BOOTTIME; case CLOCK_TAI: return HRTIMER_BASE_TAI; default: WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id); return HRTIMER_BASE_MONOTONIC; }}
static ktime_t __hrtimer_cb_get_time(clockid_t clock_id){ switch (clock_id) { case CLOCK_MONOTONIC: return ktime_get(); case CLOCK_REALTIME: return ktime_get_real(); case CLOCK_BOOTTIME: return ktime_get_boottime(); case CLOCK_TAI: return ktime_get_clocktai(); default: WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id); return ktime_get(); }}
ktime_t hrtimer_cb_get_time(const struct hrtimer *timer){ return __hrtimer_cb_get_time(timer->base->clockid);}EXPORT_SYMBOL_GPL(hrtimer_cb_get_time);
static void __hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*fn)(struct hrtimer *), clockid_t clock_id, enum hrtimer_mode mode){ bool softtimer = !!(mode & HRTIMER_MODE_SOFT); struct hrtimer_cpu_base *cpu_base; int base;
/* * On PREEMPT_RT enabled kernels hrtimers which are not explicitly * marked for hard interrupt expiry mode are moved into soft * interrupt context for latency reasons and because the callbacks * can invoke functions which might sleep on RT, e.g. spin_lock(). */ if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD)) softtimer = true;
memset(timer, 0, sizeof(struct hrtimer));
cpu_base = raw_cpu_ptr(&hrtimer_bases);
/* * POSIX magic: Relative CLOCK_REALTIME timers are not affected by * clock modifications, so they needs to become CLOCK_MONOTONIC to * ensure POSIX compliance. */ if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL) clock_id = CLOCK_MONOTONIC;
base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0; base += hrtimer_clockid_to_base(clock_id); timer->is_soft = softtimer; timer->is_hard = !!(mode & HRTIMER_MODE_HARD); timer->is_lazy = !!(mode & HRTIMER_MODE_LAZY_REARM); timer->base = &cpu_base->clock_base[base]; timerqueue_linked_init(&timer->node);
if (WARN_ON_ONCE(!fn)) ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout; else ACCESS_PRIVATE(timer, function) = fn;}
/** * hrtimer_setup - initialize a timer to the given clock * @timer: the timer to be initialized * @function: the callback function * @clock_id: the clock to be used * @mode: The modes which are relevant for initialization: * HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT, * HRTIMER_MODE_REL_SOFT * * The PINNED variants of the above can be handed in, * but the PINNED bit is ignored as pinning happens * when the hrtimer is started */void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *), clockid_t clock_id, enum hrtimer_mode mode){ debug_setup(timer, clock_id, mode); __hrtimer_setup(timer, function, clock_id, mode);}EXPORT_SYMBOL_GPL(hrtimer_setup);
/** * hrtimer_setup_on_stack - initialize a timer on stack memory * @timer: The timer to be initialized * @function: the callback function * @clock_id: The clock to be used * @mode: The timer mode * * Similar to hrtimer_setup(), except that this one must be used if struct hrtimer is in stack * memory. */void hrtimer_setup_on_stack(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *), clockid_t clock_id, enum hrtimer_mode mode){ debug_setup_on_stack(timer, clock_id, mode); __hrtimer_setup(timer, function, clock_id, mode);}EXPORT_SYMBOL_GPL(hrtimer_setup_on_stack);
/* * A timer is active, when it is enqueued into the rbtree or the * callback function is running or it's in the state of being migrated * to another cpu. * * It is important for this function to not return a false negative. */bool hrtimer_active(const struct hrtimer *timer){ struct hrtimer_clock_base *base; unsigned int seq;
do { base = READ_ONCE(timer->base); seq = raw_read_seqcount_begin(&base->seq);
if (timer->is_queued || base->running == timer) return true;
} while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base));
return false;}EXPORT_SYMBOL_GPL(hrtimer_active);
/* * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3 * distinct sections: * * - queued: the timer is queued * - callback: the timer is being ran * - post: the timer is inactive or (re)queued * * On the read side we ensure we observe timer->is_queued and cpu_base->running * from the same section, if anything changed while we looked at it, we retry. * This includes timer->base changing because sequence numbers alone are * insufficient for that. * * The sequence numbers are required because otherwise we could still observe * a false negative if the read side got smeared over multiple consecutive * __run_hrtimer() invocations. */static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_clock_base *base, struct hrtimer *timer, ktime_t now, unsigned long flags) __must_hold(&cpu_base->lock){ enum hrtimer_restart (*fn)(struct hrtimer *); bool expires_in_hardirq; int restart;
lockdep_assert_held(&cpu_base->lock);
debug_hrtimer_deactivate(timer); base->running = timer;
/* * Separate the ->running assignment from the ->is_queued assignment. * * As with a regular write barrier, this ensures the read side in * hrtimer_active() cannot observe base->running == NULL && * timer->is_queued == INACTIVE. */ raw_write_seqcount_barrier(&base->seq);
__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, false); fn = ACCESS_PRIVATE(timer, function);
/* * Clear the 'is relative' flag for the TIME_LOW_RES case. If the * timer is restarted with a period then it becomes an absolute * timer. If its not restarted it does not matter. */ if (IS_ENABLED(CONFIG_TIME_LOW_RES)) timer->is_rel = false;
/* * The timer is marked as running in the CPU base, so it is * protected against migration to a different CPU even if the lock * is dropped. */ raw_spin_unlock_irqrestore(&cpu_base->lock, flags); trace_hrtimer_expire_entry(timer, now); expires_in_hardirq = lockdep_hrtimer_enter(timer);
restart = fn(timer);
lockdep_hrtimer_exit(expires_in_hardirq); trace_hrtimer_expire_exit(timer); raw_spin_lock_irq(&cpu_base->lock);
/* * Note: We clear the running state after enqueue_hrtimer and * we do not reprogram the event hardware. Happens either in * hrtimer_start_range_ns() or in hrtimer_interrupt() * * Note: Because we dropped the cpu_base->lock above, * hrtimer_start_range_ns() can have popped in and enqueued the timer * for us already. */ if (restart == HRTIMER_RESTART && !timer->is_queued) enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false);
/* * Separate the ->running assignment from the ->is_queued assignment. * * As with a regular write barrier, this ensures the read side in * hrtimer_active() cannot observe base->running.timer == NULL && * timer->is_queued == INACTIVE. */ raw_write_seqcount_barrier(&base->seq);
WARN_ON_ONCE(base->running != timer); base->running = NULL;}
static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base){ struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
return next ? hrtimer_from_timerqueue_node(next) : NULL;}
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, unsigned long flags, unsigned int active_mask){ unsigned int active = cpu_base->active_bases & active_mask; struct hrtimer_clock_base *base;
for_each_active_base(base, cpu_base, active) { ktime_t basenow = ktime_add(now, base->offset); struct hrtimer *timer;
while ((timer = clock_base_next_timer(base))) { /* * The immediate goal for using the softexpires is * minimizing wakeups, not running timers at the * earliest interrupt after their soft expiration. * This allows us to avoid using a Priority Search * Tree, which can answer a stabbing query for * overlapping intervals and instead use the simple * BST we already have. * We don't add extra wakeups by delaying timers that * are right-of a not yet expired timer, because that * timer will have to trigger a wakeup anyway. */ if (basenow < hrtimer_get_softexpires(timer)) break;
__run_hrtimer(cpu_base, base, timer, basenow, flags); if (active_mask == HRTIMER_ACTIVE_SOFT) hrtimer_sync_wait_running(cpu_base, flags); } }}
static __latent_entropy void hrtimer_run_softirq(void){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); unsigned long flags; ktime_t now;
hrtimer_cpu_base_lock_expiry(cpu_base); raw_spin_lock_irqsave(&cpu_base->lock, flags);
now = hrtimer_update_base(cpu_base); __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
cpu_base->softirq_activated = false; hrtimer_update_softirq_timer(cpu_base, true);
raw_spin_unlock_irqrestore(&cpu_base->lock, flags); hrtimer_cpu_base_unlock_expiry(cpu_base);}
#ifdef CONFIG_HIGH_RES_TIMERS
/* * Very similar to hrtimer_force_reprogram(), except it deals with * deferred_rearm and hang_detected. */static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next, bool deferred){ cpu_base->expires_next = expires_next; cpu_base->deferred_rearm = false;
if (unlikely(cpu_base->hang_detected)) { /* * Give the system a chance to do something else than looping * on hrtimer interrupts. */ expires_next = ktime_add_ns(ktime_get(), min(100 * NSEC_PER_MSEC, cpu_base->max_hang_time)); } hrtimer_rearm_event(expires_next, deferred);}
#ifdef CONFIG_HRTIMER_REARM_DEFERREDvoid __hrtimer_rearm_deferred(void){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); ktime_t expires_next;
if (!cpu_base->deferred_rearm) return;
guard(raw_spinlock)(&cpu_base->lock); if (cpu_base->deferred_needs_update) { hrtimer_update_base(cpu_base); expires_next = hrtimer_update_next_event(cpu_base); } else { /* No timer added/removed. Use the cached value */ expires_next = cpu_base->deferred_expires_next; } hrtimer_rearm(cpu_base, expires_next, true);}
static __always_inline voidhrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next){ /* hrtimer_interrupt() just re-evaluated the first expiring timer */ cpu_base->deferred_needs_update = false; /* Cache the expiry time */ cpu_base->deferred_expires_next = expires_next; set_thread_flag(TIF_HRTIMER_REARM);}#else /* CONFIG_HRTIMER_REARM_DEFERRED */static __always_inline voidhrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next){ hrtimer_rearm(cpu_base, expires_next, false);}#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */
/* * High resolution timer interrupt * Called with interrupts disabled */void hrtimer_interrupt(struct clock_event_device *dev){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); ktime_t expires_next, now, entry_time, delta; unsigned long flags; int retries = 0;
BUG_ON(!cpu_base->hres_active); cpu_base->nr_events++; dev->next_event = KTIME_MAX; dev->next_event_forced = 0;
raw_spin_lock_irqsave(&cpu_base->lock, flags); entry_time = now = hrtimer_update_base(cpu_base);retry: cpu_base->deferred_rearm = true; /* * Set expires_next to KTIME_MAX, which prevents that remote CPUs queue * timers while __hrtimer_run_queues() is expiring the clock bases. * Timers which are re/enqueued on the local CPU are not affected by * this. */ cpu_base->expires_next = KTIME_MAX;
if (!ktime_before(now, cpu_base->softirq_expires_next)) { cpu_base->softirq_expires_next = KTIME_MAX; cpu_base->softirq_activated = true; raise_timer_softirq(HRTIMER_SOFTIRQ); }
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
/* * The next timer was already expired due to: * - tracing * - long lasting callbacks * - being scheduled away when running in a VM * * We need to prevent that we loop forever in the hrtiner interrupt * routine. We give it 3 attempts to avoid overreacting on some * spurious event. */ now = hrtimer_update_base(cpu_base); expires_next = hrtimer_update_next_event(cpu_base); cpu_base->hang_detected = false; if (expires_next < now) { if (++retries < 3) goto retry;
delta = ktime_sub(now, entry_time); cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta); cpu_base->nr_hangs++; cpu_base->hang_detected = true; }
hrtimer_interrupt_rearm(cpu_base, expires_next); raw_spin_unlock_irqrestore(&cpu_base->lock, flags);}
#endif /* !CONFIG_HIGH_RES_TIMERS */
/* * Called from run_local_timers in hardirq context every jiffy */void hrtimer_run_queues(void){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); unsigned long flags; ktime_t now;
if (hrtimer_hres_active(cpu_base)) return;
/* * This _is_ ugly: We have to check periodically, whether we * can switch to highres and / or nohz mode. The clocksource * switch happens with xtime_lock held. Notification from * there only sets the check bit in the tick_oneshot code, * otherwise we might deadlock vs. xtime_lock. */ if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) { hrtimer_switch_to_hres(); return; }
raw_spin_lock_irqsave(&cpu_base->lock, flags); now = hrtimer_update_base(cpu_base);
if (!ktime_before(now, cpu_base->softirq_expires_next)) { cpu_base->softirq_expires_next = KTIME_MAX; cpu_base->softirq_activated = true; raise_timer_softirq(HRTIMER_SOFTIRQ); }
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); raw_spin_unlock_irqrestore(&cpu_base->lock, flags);}
/* * Sleep related functions: */static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer){ struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); struct task_struct *task = t->task;
t->task = NULL; if (task) wake_up_process(task);
return HRTIMER_NORESTART;}
/** * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer * @sl: sleeper to be started * @mode: timer mode abs/rel * * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context) */void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode mode){ /* * Make the enqueue delivery mode check work on RT. If the sleeper * was initialized for hard interrupt delivery, force the mode bit. * This is a special case for hrtimer_sleepers because * __hrtimer_setup_sleeper() determines the delivery mode on RT so the * fiddling with this decision is avoided at the call sites. */ if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard) mode |= HRTIMER_MODE_HARD;
/* If already expired, clear the task pointer and set current state to running */ if (!hrtimer_start_expires_user(&sl->timer, mode)) { sl->task = NULL; __set_current_state(TASK_RUNNING); }}EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id, enum hrtimer_mode mode){ /* * On PREEMPT_RT enabled kernels hrtimers which are not explicitly * marked for hard interrupt expiry mode are moved into soft * interrupt context either for latency reasons or because the * hrtimer callback takes regular spinlocks or invokes other * functions which are not suitable for hard interrupt context on * PREEMPT_RT. * * The hrtimer_sleeper callback is RT compatible in hard interrupt * context, but there is a latency concern: Untrusted userspace can * spawn many threads which arm timers for the same expiry time on * the same CPU. That causes a latency spike due to the wakeup of * a gazillion threads. * * OTOH, privileged real-time user space applications rely on the * low latency of hard interrupt wakeups. If the current task is in * a real-time scheduling class, mark the mode for hard interrupt * expiry. */ if (IS_ENABLED(CONFIG_PREEMPT_RT)) { if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT)) mode |= HRTIMER_MODE_HARD; }
__hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode); sl->task = current;}
/** * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory * @sl: sleeper to be initialized * @clock_id: the clock to be used * @mode: timer mode abs/rel */void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id, enum hrtimer_mode mode){ debug_setup_on_stack(&sl->timer, clock_id, mode); __hrtimer_setup_sleeper(sl, clock_id, mode);}EXPORT_SYMBOL_GPL(hrtimer_setup_sleeper_on_stack);
int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts){ switch(restart->nanosleep.type) {#ifdef CONFIG_COMPAT_32BIT_TIME case TT_COMPAT: if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp)) return -EFAULT; break;#endif case TT_NATIVE: if (put_timespec64(ts, restart->nanosleep.rmtp)) return -EFAULT; break; default: BUG(); } return -ERESTART_RESTARTBLOCK;}
static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode){ struct restart_block *restart;
do { set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); hrtimer_sleeper_start_expires(t, mode);
if (likely(t->task)) schedule();
hrtimer_cancel(&t->timer); mode = HRTIMER_MODE_ABS;
} while (t->task && !signal_pending(current));
__set_current_state(TASK_RUNNING);
if (!t->task) return 0;
restart = ¤t->restart_block; if (restart->nanosleep.type != TT_NONE) { ktime_t rem = hrtimer_expires_remaining(&t->timer); struct timespec64 rmt;
if (rem <= 0) return 0; rmt = ktime_to_timespec64(rem);
return nanosleep_copyout(restart, &rmt); } return -ERESTART_RESTARTBLOCK;}
static long __sched hrtimer_nanosleep_restart(struct restart_block *restart){ struct hrtimer_sleeper t; int ret;
hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS); hrtimer_set_expires(&t.timer, restart->nanosleep.expires); ret = do_nanosleep(&t, HRTIMER_MODE_ABS); destroy_hrtimer_on_stack(&t.timer); return ret;}
long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, const clockid_t clockid){ struct restart_block *restart; struct hrtimer_sleeper t; int ret;
hrtimer_setup_sleeper_on_stack(&t, clockid, mode); hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns); ret = do_nanosleep(&t, mode); if (ret != -ERESTART_RESTARTBLOCK) goto out;
/* Absolute timers do not update the rmtp value and restart: */ if (mode == HRTIMER_MODE_ABS) { ret = -ERESTARTNOHAND; goto out; }
restart = ¤t->restart_block; restart->nanosleep.clockid = t.timer.base->clockid; restart->nanosleep.expires = hrtimer_get_expires(&t.timer); set_restart_fn(restart, hrtimer_nanosleep_restart);out: destroy_hrtimer_on_stack(&t.timer); return ret;}
#ifdef CONFIG_64BIT
SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp, struct __kernel_timespec __user *, rmtp){ struct timespec64 tu;
if (get_timespec64(&tu, rqtp)) return -EFAULT;
if (!timespec64_valid(&tu)) return -EINVAL;
current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE; current->restart_block.nanosleep.rmtp = rmtp; return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);}
#endif
#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp, struct old_timespec32 __user *, rmtp){ struct timespec64 tu;
if (get_old_timespec32(&tu, rqtp)) return -EFAULT;
if (!timespec64_valid(&tu)) return -EINVAL;
current->restart_block.fn = do_no_restart_syscall; current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE; current->restart_block.nanosleep.compat_rmtp = rmtp; return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);}#endif
/* * Functions related to boot-time initialization: */int hrtimers_prepare_cpu(unsigned int cpu){ struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
clock_b->cpu_base = cpu_base; seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock); timerqueue_linked_init_head(&clock_b->active); }
cpu_base->cpu = cpu; hrtimer_cpu_base_init_expiry_lock(cpu_base); return 0;}
int hrtimers_cpu_starting(unsigned int cpu){ struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
/* Clear out any left over state from a CPU down operation */ cpu_base->active_bases = 0; cpu_base->hres_active = false; cpu_base->hang_detected = false; cpu_base->next_timer = NULL; cpu_base->softirq_next_timer = NULL; cpu_base->expires_next = KTIME_MAX; cpu_base->softirq_expires_next = KTIME_MAX; cpu_base->softirq_activated = false; cpu_base->online = true; return 0;}
#ifdef CONFIG_HOTPLUG_CPU
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct hrtimer_clock_base *new_base){ struct timerqueue_linked_node *node; struct hrtimer *timer;
while ((node = timerqueue_linked_first(&old_base->active))) { timer = hrtimer_from_timerqueue_node(node); BUG_ON(hrtimer_callback_running(timer)); debug_hrtimer_deactivate(timer);
/* * Mark it as ENQUEUED not INACTIVE otherwise the * timer could be seen as !active and just vanish away * under us on another CPU */ __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, false); timer->base = new_base; /* * Enqueue the timers on the new cpu. This does not * reprogram the event device in case the timer * expires before the earliest on this CPU, but we run * hrtimer_interrupt after we migrated everything to * sort out already expired timers and reprogram the * event device. */ enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true); }}
int hrtimers_cpu_dying(unsigned int dying_cpu){ int ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER)); struct hrtimer_cpu_base *old_base, *new_base;
old_base = this_cpu_ptr(&hrtimer_bases); new_base = &per_cpu(hrtimer_bases, ncpu);
/* * The caller is globally serialized and nobody else * takes two locks at once, deadlock is not possible. */ raw_spin_lock(&old_base->lock); raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);
for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) migrate_hrtimer_list(&old_base->clock_base[i], &new_base->clock_base[i]);
/* Tell the other CPU to retrigger the next event */ smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);
raw_spin_unlock(&new_base->lock); old_base->online = false; raw_spin_unlock(&old_base->lock);
return 0;}
#endif /* CONFIG_HOTPLUG_CPU */
void __init hrtimers_init(void){ hrtimers_prepare_cpu(smp_processor_id()); hrtimers_cpu_starting(smp_processor_id()); open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);}