Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2012 ARM Ltd. * Author: Marc Zyngier <marc.zyngier@arm.com> */
#include <linux/cpu.h>#include <linux/kvm.h>#include <linux/kvm_host.h>#include <linux/interrupt.h>#include <linux/irq.h>#include <linux/irqdomain.h>#include <linux/uaccess.h>
#include <clocksource/arm_arch_timer.h>#include <asm/arch_timer.h>#include <asm/kvm_emulate.h>#include <asm/kvm_hyp.h>#include <asm/kvm_nested.h>
#include <kvm/arm_vgic.h>#include <kvm/arm_arch_timer.h>
#include "trace.h"
static struct timecounter *timecounter;static unsigned int host_vtimer_irq;static unsigned int host_ptimer_irq;static u32 host_vtimer_irq_flags;static u32 host_ptimer_irq_flags;
static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);DEFINE_STATIC_KEY_FALSE(broken_cntvoff_key);
static const u8 default_ppi[] = { [TIMER_PTIMER] = 30, [TIMER_VTIMER] = 27, [TIMER_HPTIMER] = 26, [TIMER_HVTIMER] = 28,};
static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, struct arch_timer_context *timer_ctx);static bool kvm_timer_pending(struct arch_timer_context *timer_ctx);static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, struct arch_timer_context *timer, enum kvm_arch_timer_regs treg, u64 val);static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, struct arch_timer_context *timer, enum kvm_arch_timer_regs treg);static bool kvm_arch_timer_get_input_level(int vintid);
static unsigned long kvm_arch_timer_get_irq_flags(void){ return kvm_vgic_global_state.no_hw_deactivation ? VGIC_IRQ_SW_RESAMPLE : 0;}
static const struct irq_ops arch_timer_irq_ops = { .get_flags = kvm_arch_timer_get_irq_flags, .get_input_level = kvm_arch_timer_get_input_level,};
static const struct irq_ops arch_timer_irq_ops_vgic_v5 = { .get_input_level = kvm_arch_timer_get_input_level, .queue_irq_unlock = vgic_v5_ppi_queue_irq_unlock, .set_direct_injection = vgic_v5_set_ppi_dvi,};
static int nr_timers(struct kvm_vcpu *vcpu){ if (!vcpu_has_nv(vcpu)) return NR_KVM_EL0_TIMERS;
return NR_KVM_TIMERS;}
u32 timer_get_ctl(struct arch_timer_context *ctxt){ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
switch(arch_timer_ctx_index(ctxt)) { case TIMER_VTIMER: return __vcpu_sys_reg(vcpu, CNTV_CTL_EL0); case TIMER_PTIMER: return __vcpu_sys_reg(vcpu, CNTP_CTL_EL0); case TIMER_HVTIMER: return __vcpu_sys_reg(vcpu, CNTHV_CTL_EL2); case TIMER_HPTIMER: return __vcpu_sys_reg(vcpu, CNTHP_CTL_EL2); default: WARN_ON(1); return 0; }}
u64 timer_get_cval(struct arch_timer_context *ctxt){ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
switch(arch_timer_ctx_index(ctxt)) { case TIMER_VTIMER: return __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); case TIMER_PTIMER: return __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); case TIMER_HVTIMER: return __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2); case TIMER_HPTIMER: return __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); default: WARN_ON(1); return 0; }}
static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl){ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
switch(arch_timer_ctx_index(ctxt)) { case TIMER_VTIMER: __vcpu_assign_sys_reg(vcpu, CNTV_CTL_EL0, ctl); break; case TIMER_PTIMER: __vcpu_assign_sys_reg(vcpu, CNTP_CTL_EL0, ctl); break; case TIMER_HVTIMER: __vcpu_assign_sys_reg(vcpu, CNTHV_CTL_EL2, ctl); break; case TIMER_HPTIMER: __vcpu_assign_sys_reg(vcpu, CNTHP_CTL_EL2, ctl); break; default: WARN_ON(1); }}
static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval){ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
switch(arch_timer_ctx_index(ctxt)) { case TIMER_VTIMER: __vcpu_assign_sys_reg(vcpu, CNTV_CVAL_EL0, cval); break; case TIMER_PTIMER: __vcpu_assign_sys_reg(vcpu, CNTP_CVAL_EL0, cval); break; case TIMER_HVTIMER: __vcpu_assign_sys_reg(vcpu, CNTHV_CVAL_EL2, cval); break; case TIMER_HPTIMER: __vcpu_assign_sys_reg(vcpu, CNTHP_CVAL_EL2, cval); break; default: WARN_ON(1); }}
u64 kvm_phys_timer_read(void){ return timecounter->cc->read(timecounter->cc);}
void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map){ if (vcpu_has_nv(vcpu)) { if (is_hyp_ctxt(vcpu)) { map->direct_vtimer = vcpu_hvtimer(vcpu); map->direct_ptimer = vcpu_hptimer(vcpu); map->emul_vtimer = vcpu_vtimer(vcpu); map->emul_ptimer = vcpu_ptimer(vcpu); } else { map->direct_vtimer = vcpu_vtimer(vcpu); map->direct_ptimer = vcpu_ptimer(vcpu); map->emul_vtimer = vcpu_hvtimer(vcpu); map->emul_ptimer = vcpu_hptimer(vcpu); } } else if (has_vhe()) { map->direct_vtimer = vcpu_vtimer(vcpu); map->direct_ptimer = vcpu_ptimer(vcpu); map->emul_vtimer = NULL; map->emul_ptimer = NULL; } else { map->direct_vtimer = vcpu_vtimer(vcpu); map->direct_ptimer = NULL; map->emul_vtimer = NULL; map->emul_ptimer = vcpu_ptimer(vcpu); }
trace_kvm_get_timer_map(vcpu->vcpu_id, map);}
static inline bool userspace_irqchip(struct kvm *kvm){ return unlikely(!irqchip_in_kernel(kvm));}
static void soft_timer_start(struct hrtimer *hrt, u64 ns){ hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns), HRTIMER_MODE_ABS_HARD);}
static void soft_timer_cancel(struct hrtimer *hrt){ hrtimer_cancel(hrt);}
static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id){ struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id; struct arch_timer_context *ctx; struct timer_map map;
/* * We may see a timer interrupt after vcpu_put() has been called which * sets the CPU's vcpu pointer to NULL, because even though the timer * has been disabled in timer_save_state(), the hardware interrupt * signal may not have been retired from the interrupt controller yet. */ if (!vcpu) return IRQ_HANDLED;
get_timer_map(vcpu, &map);
if (irq == host_vtimer_irq) ctx = map.direct_vtimer; else ctx = map.direct_ptimer;
if (kvm_timer_pending(ctx)) kvm_timer_update_irq(vcpu, true, ctx);
if (userspace_irqchip(vcpu->kvm) && !static_branch_unlikely(&has_gic_active_state)) disable_percpu_irq(host_vtimer_irq);
return IRQ_HANDLED;}
static u64 kvm_counter_compute_delta(struct arch_timer_context *timer_ctx, u64 val){ u64 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
if (now < val) { u64 ns;
ns = cyclecounter_cyc2ns(timecounter->cc, val - now, timecounter->mask, &timer_ctx->ns_frac); return ns; }
return 0;}
static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx){ return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx));}
static bool kvm_timer_enabled(struct arch_timer_context *timer_ctx){ WARN_ON(timer_ctx && timer_ctx->loaded); return timer_ctx && ((timer_get_ctl(timer_ctx) & (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE);}
static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu){ return (cpus_have_final_cap(ARM64_HAS_WFXT) && vcpu_get_flag(vcpu, IN_WFIT));}
static u64 wfit_delay_ns(struct kvm_vcpu *vcpu){ u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); struct arch_timer_context *ctx;
ctx = is_hyp_ctxt(vcpu) ? vcpu_hvtimer(vcpu) : vcpu_vtimer(vcpu);
return kvm_counter_compute_delta(ctx, val);}
/* * Returns the earliest expiration time in ns among guest timers. * Note that it will return 0 if none of timers can fire. */static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu){ u64 min_delta = ULLONG_MAX; int i;
for (i = 0; i < nr_timers(vcpu); i++) { struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
WARN(ctx->loaded, "timer %d loaded\n", i); if (kvm_timer_enabled(ctx)) min_delta = min(min_delta, kvm_timer_compute_delta(ctx)); }
if (vcpu_has_wfit_active(vcpu)) min_delta = min(min_delta, wfit_delay_ns(vcpu));
/* If none of timers can fire, then return 0 */ if (min_delta == ULLONG_MAX) return 0;
return min_delta;}
static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt){ struct arch_timer_cpu *timer; struct kvm_vcpu *vcpu; u64 ns;
timer = container_of(hrt, struct arch_timer_cpu, bg_timer); vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
/* * Check that the timer has really expired from the guest's * PoV (NTP on the host may have forced it to expire * early). If we should have slept longer, restart it. */ ns = kvm_timer_earliest_exp(vcpu); if (unlikely(ns)) { hrtimer_forward_now(hrt, ns_to_ktime(ns)); return HRTIMER_RESTART; }
kvm_vcpu_wake_up(vcpu); return HRTIMER_NORESTART;}
static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt){ struct arch_timer_context *ctx; struct kvm_vcpu *vcpu; u64 ns;
ctx = container_of(hrt, struct arch_timer_context, hrtimer); vcpu = timer_context_to_vcpu(ctx);
trace_kvm_timer_hrtimer_expire(ctx);
/* * Check that the timer has really expired from the guest's * PoV (NTP on the host may have forced it to expire * early). If not ready, schedule for a later time. */ ns = kvm_timer_compute_delta(ctx); if (unlikely(ns)) { hrtimer_forward_now(hrt, ns_to_ktime(ns)); return HRTIMER_RESTART; }
kvm_timer_update_irq(vcpu, true, ctx); return HRTIMER_NORESTART;}
static bool kvm_timer_pending(struct arch_timer_context *timer_ctx){ enum kvm_arch_timers index; u64 cval, now;
if (!timer_ctx) return false;
index = arch_timer_ctx_index(timer_ctx);
if (timer_ctx->loaded) { u32 cnt_ctl = 0;
switch (index) { case TIMER_VTIMER: case TIMER_HVTIMER: cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL); break; case TIMER_PTIMER: case TIMER_HPTIMER: cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL); break; case NR_KVM_TIMERS: /* GCC is braindead */ cnt_ctl = 0; break; }
return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) && (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) && !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK); }
if (!kvm_timer_enabled(timer_ctx)) return false;
cval = timer_get_cval(timer_ctx); now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
return cval <= now;}
int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu){ return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;}
static u64 kvm_timer_needs_notify(struct kvm_vcpu *vcpu){ u64 v = vcpu->run->s.regs.device_irq_level;
v ^= kvm_timer_pending(vcpu_vtimer(vcpu)) ? KVM_ARM_DEV_EL1_VTIMER : 0; v ^= kvm_timer_pending(vcpu_ptimer(vcpu)) ? KVM_ARM_DEV_EL1_PTIMER : 0;
return v & (KVM_ARM_DEV_EL1_VTIMER | KVM_ARM_DEV_EL1_PTIMER);}
bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu){ return !!kvm_timer_needs_notify(vcpu);}
/* * Reflect the timer output level into the kvm_run structure */bool kvm_timer_update_run(struct kvm_vcpu *vcpu){ u64 mask = kvm_timer_needs_notify(vcpu); if (mask) vcpu->run->s.regs.device_irq_level ^= mask; return !!mask;}
static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level){ /* * Paper over NV2 brokenness by publishing the interrupt status * bit. This still results in a poor quality of emulation (guest * writes will have no effect until the next exit). * * But hey, it's fast, right? */ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx); if (is_hyp_ctxt(vcpu) && (ctx == vcpu_vtimer(vcpu) || ctx == vcpu_ptimer(vcpu))) { unsigned long val = timer_get_ctl(ctx); __assign_bit(__ffs(ARCH_TIMER_CTRL_IT_STAT), &val, level); timer_set_ctl(ctx, val); }}
static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, struct arch_timer_context *timer_ctx){ kvm_timer_update_status(timer_ctx, new_level);
trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx), new_level);
if (userspace_irqchip(vcpu->kvm)) return;
/* Skip injecting on GICv5 for directly injected (DVI'd) timers */ if (vgic_is_v5(vcpu->kvm)) { struct timer_map map;
get_timer_map(vcpu, &map);
if (map.direct_ptimer == timer_ctx || map.direct_vtimer == timer_ctx) return; }
kvm_vgic_inject_irq(vcpu->kvm, vcpu, timer_irq(timer_ctx), new_level, timer_ctx);}
/* Only called for a fully emulated timer */static void timer_emulate(struct arch_timer_context *ctx){ bool pending = kvm_timer_pending(ctx);
trace_kvm_timer_emulate(ctx, pending);
kvm_timer_update_irq(timer_context_to_vcpu(ctx), pending, ctx);
/* * If the timer is pending, we don't need to have a soft timer * scheduled for the future. If the timer is disabled, then * we don't need a soft timer either. */ if (pending || !kvm_timer_enabled(ctx)) return;
soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));}
static void set_cntvoff(u64 cntvoff){ kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff);}
static void set_cntpoff(u64 cntpoff){ if (has_cntpoff()) write_sysreg_s(cntpoff, SYS_CNTPOFF_EL2);}
static void timer_save_state(struct arch_timer_context *ctx){ struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx)); enum kvm_arch_timers index = arch_timer_ctx_index(ctx); unsigned long flags;
if (!timer->enabled) return;
local_irq_save(flags);
if (!ctx->loaded) goto out;
switch (index) { u64 cval;
case TIMER_VTIMER: case TIMER_HVTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL)); cval = read_sysreg_el0(SYS_CNTV_CVAL);
if (has_broken_cntvoff()) cval -= timer_get_offset(ctx);
timer_set_cval(ctx, cval);
/* Disable the timer */ write_sysreg_el0(0, SYS_CNTV_CTL); isb();
/* * The kernel may decide to run userspace after * calling vcpu_put, so we reset cntvoff to 0 to * ensure a consistent read between user accesses to * the virtual counter and kernel access to the * physical counter of non-VHE case. * * For VHE, the virtual counter uses a fixed virtual * offset of zero, so no need to zero CNTVOFF_EL2 * register, but this is actually useful when switching * between EL1/vEL2 with NV. * * Do it unconditionally, as this is either unavoidable * or dirt cheap. */ set_cntvoff(0); break; case TIMER_PTIMER: case TIMER_HPTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL)); cval = read_sysreg_el0(SYS_CNTP_CVAL);
cval -= timer_get_offset(ctx);
timer_set_cval(ctx, cval);
/* Disable the timer */ write_sysreg_el0(0, SYS_CNTP_CTL); isb();
set_cntpoff(0); break; case NR_KVM_TIMERS: BUG(); }
trace_kvm_timer_save_state(ctx);
ctx->loaded = false;out: local_irq_restore(flags);}
/* * Schedule the background timer before calling kvm_vcpu_halt, so that this * thread is removed from its waitqueue and made runnable when there's a timer * interrupt to handle. */static void kvm_timer_blocking(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map;
get_timer_map(vcpu, &map);
/* * If no timers are capable of raising interrupts (disabled or * masked), then there's no more work for us to do. */ if (!kvm_timer_enabled(map.direct_vtimer) && !kvm_timer_enabled(map.direct_ptimer) && !kvm_timer_enabled(map.emul_vtimer) && !kvm_timer_enabled(map.emul_ptimer) && !vcpu_has_wfit_active(vcpu)) return;
/* * At least one guest time will expire. Schedule a background timer. * Set the earliest expiration time among the guest timers. */ soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));}
static void kvm_timer_unblocking(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu);
soft_timer_cancel(&timer->bg_timer);}
static void timer_restore_state(struct arch_timer_context *ctx){ struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx)); enum kvm_arch_timers index = arch_timer_ctx_index(ctx); unsigned long flags;
if (!timer->enabled) return;
local_irq_save(flags);
if (ctx->loaded) goto out;
switch (index) { u64 cval, offset;
case TIMER_VTIMER: case TIMER_HVTIMER: cval = timer_get_cval(ctx); offset = timer_get_offset(ctx); if (has_broken_cntvoff()) { set_cntvoff(0); cval += offset; } else { set_cntvoff(offset); } write_sysreg_el0(cval, SYS_CNTV_CVAL); isb(); write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTV_CTL); break; case TIMER_PTIMER: case TIMER_HPTIMER: cval = timer_get_cval(ctx); offset = timer_get_offset(ctx); set_cntpoff(offset); cval += offset; write_sysreg_el0(cval, SYS_CNTP_CVAL); isb(); write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL); break; case NR_KVM_TIMERS: BUG(); }
trace_kvm_timer_restore_state(ctx);
ctx->loaded = true;out: local_irq_restore(flags);}
static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active){ int r; r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active); WARN_ON(r);}
static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx){ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx); bool pending = kvm_timer_pending(ctx); bool phys_active = false;
/* * Update the timer output so that it is likely to match the * state we're about to restore. If the timer expires between * this point and the register restoration, we'll take the * interrupt anyway. */ kvm_timer_update_irq(vcpu, pending, ctx);
if (irqchip_in_kernel(vcpu->kvm)) phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
phys_active |= pending; phys_active |= vgic_is_v5(vcpu->kvm);
set_timer_irq_phys_active(ctx, phys_active);}
static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu){ struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); bool pending = kvm_timer_pending(vtimer);
/* * Update the timer output so that it is likely to match the * state we're about to restore. If the timer expires between * this point and the register restoration, we'll take the * interrupt anyway. */ kvm_timer_update_irq(vcpu, pending, vtimer);
/* * When using a userspace irqchip with the architected timers and a * host interrupt controller that doesn't support an active state, we * must still prevent continuously exiting from the guest, and * therefore mask the physical interrupt by disabling it on the host * interrupt controller when the virtual level is high, such that the * guest can make forward progress. Once we detect the output level * being de-asserted, we unmask the interrupt again so that we exit * from the guest when the timer fires. */ if (pending) disable_percpu_irq(host_vtimer_irq); else enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);}
/* If _pred is true, set bit in _set, otherwise set it in _clr */#define assign_clear_set_bit(_pred, _bit, _clr, _set) \ do { \ if (_pred) \ (_set) |= (_bit); \ else \ (_clr) |= (_bit); \ } while (0)
static void kvm_timer_vcpu_load_nested_switch(struct kvm_vcpu *vcpu, struct timer_map *map){ int hw, ret;
if (!irqchip_in_kernel(vcpu->kvm)) return;
/* * We only ever unmap the vtimer irq on a VHE system that runs nested * virtualization, in which case we have both a valid emul_vtimer, * emul_ptimer, direct_vtimer, and direct_ptimer. * * Since this is called from kvm_timer_vcpu_load(), a change between * vEL2 and vEL1/0 will have just happened, and the timer_map will * represent this, and therefore we switch the emul/direct mappings * below. */ hw = kvm_vgic_get_map(vcpu, timer_irq(map->direct_vtimer)); if (hw < 0) { kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_vtimer)); kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_ptimer));
ret = kvm_vgic_map_phys_irq(vcpu, map->direct_vtimer->host_timer_irq, timer_irq(map->direct_vtimer)); WARN_ON_ONCE(ret); ret = kvm_vgic_map_phys_irq(vcpu, map->direct_ptimer->host_timer_irq, timer_irq(map->direct_ptimer)); WARN_ON_ONCE(ret); }}
static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map){ bool tvt, tpt, tvc, tpc, tvt02, tpt02; u64 clr, set;
/* * No trapping gets configured here with nVHE. See * __timer_enable_traps(), which is where the stuff happens. */ if (!has_vhe()) return;
/* * Our default policy is not to trap anything. As we progress * within this function, reality kicks in and we start adding * traps based on emulation requirements. */ tvt = tpt = tvc = tpc = false; tvt02 = tpt02 = false;
/* * NV2 badly breaks the timer semantics by redirecting accesses to * the EL1 timer state to memory, so let's call ECV to the rescue if * available: we trap all CNT{P,V}_{CTL,CVAL,TVAL}_EL0 accesses. * * The treatment slightly varies depending whether we run a nVHE or * VHE guest: nVHE will use the _EL0 registers directly, while VHE * will use the _EL02 accessors. This translates in different trap * bits. * * None of the trapping is required when running in non-HYP context, * unless required by the L1 hypervisor settings once we advertise * ECV+NV in the guest, or that we need trapping for other reasons. */ if (cpus_have_final_cap(ARM64_HAS_ECV) && is_hyp_ctxt(vcpu)) { if (vcpu_el2_e2h_is_set(vcpu)) tvt02 = tpt02 = true; else tvt = tpt = true; }
/* * We have two possibility to deal with a physical offset: * * - Either we have CNTPOFF (yay!) or the offset is 0: * we let the guest freely access the HW * * - or neither of these condition apply: * we trap accesses to the HW, but still use it * after correcting the physical offset */ if (!has_cntpoff() && timer_get_offset(map->direct_ptimer)) tpt = tpc = true;
/* * For the poor sods that could not correctly subtract one value * from another, trap the full virtual timer and counter. */ if (has_broken_cntvoff() && timer_get_offset(map->direct_vtimer)) tvt = tvc = true;
/* * Apply the enable bits that the guest hypervisor has requested for * its own guest. We can only add traps that wouldn't have been set * above. * Implementation choices: we do not support NV when E2H=0 in the * guest, and we don't support configuration where E2H is writable * by the guest (either FEAT_VHE or FEAT_E2H0 is implemented, but * not both). This simplifies the handling of the EL1NV* bits. */ if (is_nested_ctxt(vcpu)) { u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
/* Use the VHE format for mental sanity */ if (!vcpu_el2_e2h_is_set(vcpu)) val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
tpt |= !(val & (CNTHCTL_EL1PCEN << 10)); tpc |= !(val & (CNTHCTL_EL1PCTEN << 10));
tpt02 |= (val & CNTHCTL_EL1NVPCT); tvt02 |= (val & CNTHCTL_EL1NVVCT); }
/* * Now that we have collected our requirements, compute the * trap and enable bits. */ set = 0; clr = 0;
assign_clear_set_bit(tpt, CNTHCTL_EL1PCEN << 10, set, clr); assign_clear_set_bit(tpc, CNTHCTL_EL1PCTEN << 10, set, clr); assign_clear_set_bit(tvt, CNTHCTL_EL1TVT, clr, set); assign_clear_set_bit(tvc, CNTHCTL_EL1TVCT, clr, set); assign_clear_set_bit(tvt02, CNTHCTL_EL1NVVCT, clr, set); assign_clear_set_bit(tpt02, CNTHCTL_EL1NVPCT, clr, set);
/* This only happens on VHE, so use the CNTHCTL_EL2 accessor. */ sysreg_clear_set(cnthctl_el2, clr, set);}
void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map;
if (unlikely(!timer->enabled)) return;
get_timer_map(vcpu, &map);
if (static_branch_likely(&has_gic_active_state)) { /* We don't do NV on GICv5, yet */ if (vcpu_has_nv(vcpu) && !vgic_is_v5(vcpu->kvm)) kvm_timer_vcpu_load_nested_switch(vcpu, &map);
kvm_timer_vcpu_load_gic(map.direct_vtimer); if (map.direct_ptimer) kvm_timer_vcpu_load_gic(map.direct_ptimer); } else { kvm_timer_vcpu_load_nogic(vcpu); }
kvm_timer_unblocking(vcpu);
timer_restore_state(map.direct_vtimer); if (map.direct_ptimer) timer_restore_state(map.direct_ptimer); if (map.emul_vtimer) timer_emulate(map.emul_vtimer); if (map.emul_ptimer) timer_emulate(map.emul_ptimer);
timer_set_traps(vcpu, &map);}
void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map;
if (unlikely(!timer->enabled)) return;
get_timer_map(vcpu, &map);
timer_save_state(map.direct_vtimer); if (map.direct_ptimer) timer_save_state(map.direct_ptimer);
/* * Cancel soft timer emulation, because the only case where we * need it after a vcpu_put is in the context of a sleeping VCPU, and * in that case we already factor in the deadline for the physical * timer when scheduling the bg_timer. * * In any case, we re-schedule the hrtimer for the physical timer when * coming back to the VCPU thread in kvm_timer_vcpu_load(). */ if (map.emul_vtimer) soft_timer_cancel(&map.emul_vtimer->hrtimer); if (map.emul_ptimer) soft_timer_cancel(&map.emul_ptimer->hrtimer);
if (kvm_vcpu_is_blocking(vcpu)) kvm_timer_blocking(vcpu);
if (vgic_is_v5(vcpu->kvm)) { set_timer_irq_phys_active(map.direct_vtimer, false); if (map.direct_ptimer) set_timer_irq_phys_active(map.direct_ptimer, false); }}
void kvm_timer_sync_nested(struct kvm_vcpu *vcpu){ /* * When NV2 is on, guest hypervisors have their EL1 timer register * accesses redirected to the VNCR page. Any guest action taken on * the timer is postponed until the next exit, leading to a very * poor quality of emulation. * * This is an unmitigated disaster, only papered over by FEAT_ECV, * which allows trapping of the timer registers even with NV2. * Still, this is still worse than FEAT_NV on its own. Meh. */ if (!cpus_have_final_cap(ARM64_HAS_ECV)) { /* * For a VHE guest hypervisor, the EL2 state is directly * stored in the host EL1 timers, while the emulated EL1 * state is stored in the VNCR page. The latter could have * been updated behind our back, and we must reset the * emulation of the timers. * * A non-VHE guest hypervisor doesn't have any direct access * to its timers: the EL2 registers trap despite being * notionally direct (we use the EL1 HW, as for VHE), while * the EL1 registers access memory. * * In both cases, process the emulated timers on each guest * exit. Boo. */ struct timer_map map; get_timer_map(vcpu, &map);
soft_timer_cancel(&map.emul_vtimer->hrtimer); soft_timer_cancel(&map.emul_ptimer->hrtimer); timer_emulate(map.emul_vtimer); timer_emulate(map.emul_ptimer); }}
/* * With a userspace irqchip we have to check if the guest de-asserted the * timer and if so, unmask the timer irq signal on the host interrupt * controller to ensure that we see future timer signals. */static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu){ struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
if (!kvm_timer_pending(vtimer)) { kvm_timer_update_irq(vcpu, false, vtimer); if (static_branch_likely(&has_gic_active_state)) set_timer_irq_phys_active(vtimer, false); else enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); }}
void kvm_timer_sync_user(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu);
if (unlikely(!timer->enabled)) return;
if (unlikely(!irqchip_in_kernel(vcpu->kvm))) unmask_vtimer_irq_user(vcpu);}
void kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map;
get_timer_map(vcpu, &map);
/* * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8 * and to 0 for ARMv7. We provide an implementation that always * resets the timer to be disabled and unmasked and is compliant with * the ARMv7 architecture. */ for (int i = 0; i < nr_timers(vcpu); i++) timer_set_ctl(vcpu_get_timer(vcpu, i), 0);
/* * A vcpu running at EL2 is in charge of the offset applied to * the virtual timer, so use the physical VM offset, and point * the vcpu offset to CNTVOFF_EL2. */ if (vcpu_has_nv(vcpu)) { struct arch_timer_offset *offs = &vcpu_vtimer(vcpu)->offset;
offs->vcpu_offset = __ctxt_sys_reg(&vcpu->arch.ctxt, CNTVOFF_EL2); offs->vm_offset = &vcpu->kvm->arch.timer_data.poffset; }
if (timer->enabled) { for (int i = 0; i < nr_timers(vcpu); i++) kvm_timer_update_irq(vcpu, false, vcpu_get_timer(vcpu, i));
if (irqchip_in_kernel(vcpu->kvm)) { kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_vtimer)); if (map.direct_ptimer) kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_ptimer)); } }
if (map.emul_vtimer) soft_timer_cancel(&map.emul_vtimer->hrtimer); if (map.emul_ptimer) soft_timer_cancel(&map.emul_ptimer->hrtimer);}
static void timer_context_init(struct kvm_vcpu *vcpu, int timerid){ struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid); struct kvm *kvm = vcpu->kvm;
ctxt->timer_id = timerid;
if (!kvm_vm_is_protected(vcpu->kvm)) { if (timerid == TIMER_VTIMER) ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset; else ctxt->offset.vm_offset = &kvm->arch.timer_data.poffset; } else { ctxt->offset.vm_offset = NULL; }
hrtimer_setup(&ctxt->hrtimer, kvm_hrtimer_expire, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
switch (timerid) { case TIMER_PTIMER: case TIMER_HPTIMER: ctxt->host_timer_irq = host_ptimer_irq; break; case TIMER_VTIMER: case TIMER_HVTIMER: ctxt->host_timer_irq = host_vtimer_irq; break; }}
void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu);
for (int i = 0; i < NR_KVM_TIMERS; i++) timer_context_init(vcpu, i);
/* Synchronize offsets across timers of a VM if not already provided */ if (!vcpu_is_protected(vcpu) && !test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read()); timer_set_offset(vcpu_ptimer(vcpu), 0); }
hrtimer_setup(&timer->bg_timer, kvm_bg_timer_expire, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);}
/* * This is always called during kvm_arch_init_vm, but will also be * called from kvm_vgic_create if we have a vGICv5. */void kvm_timer_init_vm(struct kvm *kvm){ /* * Set up the default PPIs - note that we adjust them based on * the model of the GIC as GICv5 uses a different way to * describing interrupts. */ for (int i = 0; i < NR_KVM_TIMERS; i++) kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]);}
void kvm_timer_cpu_up(void){ enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); if (host_ptimer_irq) enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);}
void kvm_timer_cpu_down(void){ disable_percpu_irq(host_vtimer_irq); if (host_ptimer_irq) disable_percpu_irq(host_ptimer_irq);}
static u64 read_timer_ctl(struct arch_timer_context *timer){ /* * Set ISTATUS bit if it's expired. * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit * regardless of ENABLE bit for our implementation convenience. */ u32 ctl = timer_get_ctl(timer);
if (!kvm_timer_compute_delta(timer)) ctl |= ARCH_TIMER_CTRL_IT_STAT;
return ctl;}
static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, struct arch_timer_context *timer, enum kvm_arch_timer_regs treg){ u64 val;
switch (treg) { case TIMER_REG_TVAL: val = timer_get_cval(timer) - kvm_phys_timer_read() + timer_get_offset(timer); val = lower_32_bits(val); break;
case TIMER_REG_CTL: val = read_timer_ctl(timer); break;
case TIMER_REG_CVAL: val = timer_get_cval(timer); break;
case TIMER_REG_CNT: val = kvm_phys_timer_read() - timer_get_offset(timer); break;
case TIMER_REG_VOFF: val = *timer->offset.vcpu_offset; break;
default: BUG(); }
return val;}
u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu, enum kvm_arch_timers tmr, enum kvm_arch_timer_regs treg){ struct arch_timer_context *timer; struct timer_map map; u64 val;
get_timer_map(vcpu, &map); timer = vcpu_get_timer(vcpu, tmr);
if (timer == map.emul_vtimer || timer == map.emul_ptimer) return kvm_arm_timer_read(vcpu, timer, treg);
preempt_disable(); timer_save_state(timer);
val = kvm_arm_timer_read(vcpu, timer, treg);
timer_restore_state(timer); preempt_enable();
return val;}
static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, struct arch_timer_context *timer, enum kvm_arch_timer_regs treg, u64 val){ switch (treg) { case TIMER_REG_TVAL: timer_set_cval(timer, kvm_phys_timer_read() - timer_get_offset(timer) + (s32)val); break;
case TIMER_REG_CTL: timer_set_ctl(timer, val & ~ARCH_TIMER_CTRL_IT_STAT); break;
case TIMER_REG_CVAL: timer_set_cval(timer, val); break;
case TIMER_REG_VOFF: *timer->offset.vcpu_offset = val; break;
default: BUG(); }}
void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu, enum kvm_arch_timers tmr, enum kvm_arch_timer_regs treg, u64 val){ struct arch_timer_context *timer; struct timer_map map;
get_timer_map(vcpu, &map); timer = vcpu_get_timer(vcpu, tmr); if (timer == map.emul_vtimer || timer == map.emul_ptimer) { soft_timer_cancel(&timer->hrtimer); kvm_arm_timer_write(vcpu, timer, treg, val); timer_emulate(timer); } else { preempt_disable(); timer_save_state(timer); kvm_arm_timer_write(vcpu, timer, treg, val); timer_restore_state(timer); preempt_enable(); }}
static int timer_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu){ if (vcpu) irqd_set_forwarded_to_vcpu(d); else irqd_clr_forwarded_to_vcpu(d);
return 0;}
static int timer_irq_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which, bool val){ bool passthrough = which != IRQCHIP_STATE_ACTIVE || !irqd_is_forwarded_to_vcpu(d) || (kvm_vgic_global_state.type == VGIC_V5 && vgic_is_v3(kvm_get_running_vcpu()->kvm));
if (passthrough) return irq_chip_set_parent_state(d, which, val);
if (val) irq_chip_mask_parent(d); else irq_chip_unmask_parent(d);
return 0;}
static void timer_irq_eoi(struct irq_data *d){ if (!irqd_is_forwarded_to_vcpu(d)) irq_chip_eoi_parent(d);}
static void timer_irq_ack(struct irq_data *d){ d = d->parent_data; if (d->chip->irq_ack) d->chip->irq_ack(d);}
static struct irq_chip timer_chip = { .name = "KVM", .irq_ack = timer_irq_ack, .irq_mask = irq_chip_mask_parent, .irq_unmask = irq_chip_unmask_parent, .irq_eoi = timer_irq_eoi, .irq_set_type = irq_chip_set_type_parent, .irq_set_vcpu_affinity = timer_irq_set_vcpu_affinity, .irq_set_irqchip_state = timer_irq_set_irqchip_state,};
static int timer_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg){ irq_hw_number_t hwirq = (uintptr_t)arg;
return irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &timer_chip, NULL);}
static void timer_irq_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs){}
static const struct irq_domain_ops timer_domain_ops = { .alloc = timer_irq_domain_alloc, .free = timer_irq_domain_free,};
static void kvm_irq_fixup_flags(unsigned int virq, u32 *flags){ *flags = irq_get_trigger_type(virq); if (*flags != IRQF_TRIGGER_HIGH && *flags != IRQF_TRIGGER_LOW) { kvm_err("Invalid trigger for timer IRQ%d, assuming level low\n", virq); *flags = IRQF_TRIGGER_LOW; }}
static int kvm_irq_init(struct arch_timer_kvm_info *info){ struct irq_domain *domain = NULL;
if (info->virtual_irq <= 0) { kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", info->virtual_irq); return -ENODEV; }
host_vtimer_irq = info->virtual_irq; kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags);
if (kvm_vgic_global_state.no_hw_deactivation || kvm_vgic_global_state.type == VGIC_V5) { struct fwnode_handle *fwnode; struct irq_data *data;
fwnode = irq_domain_alloc_named_fwnode("kvm-timer"); if (!fwnode) return -ENOMEM;
/* Assume both vtimer and ptimer in the same parent */ data = irq_get_irq_data(host_vtimer_irq); domain = irq_domain_create_hierarchy(data->domain, 0, NR_KVM_TIMERS, fwnode, &timer_domain_ops, NULL); if (!domain) { irq_domain_free_fwnode(fwnode); return -ENOMEM; }
WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq, (void *)TIMER_VTIMER)); }
if (info->physical_irq > 0) { host_ptimer_irq = info->physical_irq; kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags);
if (domain) WARN_ON(irq_domain_push_irq(domain, host_ptimer_irq, (void *)TIMER_PTIMER)); }
return 0;}
static void kvm_timer_handle_errata(void){ u64 mmfr0, mmfr1, mmfr4;
/* * CNTVOFF_EL2 is broken on some implementations. For those, we trap * all virtual timer/counter accesses, requiring FEAT_ECV. * * However, a hypervisor supporting nesting is likely to mitigate the * erratum at L0, and not require other levels to mitigate it (which * would otherwise be a terrible performance sink due to trap * amplification). * * Given that the affected HW implements both FEAT_VHE and FEAT_E2H0, * and that NV is likely not to (because of limitations of the * architecture), only enable the workaround when FEAT_VHE and * FEAT_E2H0 are both detected. Time will tell if this actually holds. */ mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); mmfr4 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR4_EL1); if (SYS_FIELD_GET(ID_AA64MMFR1_EL1, VH, mmfr1) && !SYS_FIELD_GET(ID_AA64MMFR4_EL1, E2H0, mmfr4) && SYS_FIELD_GET(ID_AA64MMFR0_EL1, ECV, mmfr0) && (has_vhe() || has_hvhe()) && cpus_have_final_cap(ARM64_WORKAROUND_QCOM_ORYON_CNTVOFF)) { static_branch_enable(&broken_cntvoff_key); kvm_info("Broken CNTVOFF_EL2, trapping virtual timer\n"); }}
int __init kvm_timer_hyp_init(bool has_gic){ struct arch_timer_kvm_info *info; int err;
info = arch_timer_get_kvm_info(); timecounter = &info->timecounter;
if (!timecounter->cc) { kvm_err("kvm_arch_timer: uninitialized timecounter\n"); return -ENODEV; }
err = kvm_irq_init(info); if (err) return err;
/* First, do the virtual EL1 timer irq */
err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler, "kvm guest vtimer", kvm_get_running_vcpus()); if (err) { kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n", host_vtimer_irq, err); return err; }
if (has_gic) { err = irq_set_vcpu_affinity(host_vtimer_irq, kvm_get_running_vcpus()); if (err) { kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); goto out_free_vtimer_irq; }
static_branch_enable(&has_gic_active_state); }
kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
/* Now let's do the physical EL1 timer irq */
if (info->physical_irq > 0) { err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler, "kvm guest ptimer", kvm_get_running_vcpus()); if (err) { kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n", host_ptimer_irq, err); goto out_free_vtimer_irq; }
if (has_gic) { err = irq_set_vcpu_affinity(host_ptimer_irq, kvm_get_running_vcpus()); if (err) { kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); goto out_free_ptimer_irq; } }
kvm_debug("physical timer IRQ%d\n", host_ptimer_irq); } else if (has_vhe()) { kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n", info->physical_irq); err = -ENODEV; goto out_free_vtimer_irq; }
kvm_timer_handle_errata(); return 0;
out_free_ptimer_irq: if (info->physical_irq > 0) free_percpu_irq(host_ptimer_irq, kvm_get_running_vcpus());out_free_vtimer_irq: free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus()); return err;}
void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu);
soft_timer_cancel(&timer->bg_timer);}
static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu){ u32 ppis = 0; bool valid;
mutex_lock(&vcpu->kvm->arch.config_lock);
for (int i = 0; i < nr_timers(vcpu); i++) { struct arch_timer_context *ctx; int irq;
ctx = vcpu_get_timer(vcpu, i); irq = timer_irq(ctx); if (kvm_vgic_set_owner(vcpu, irq, ctx)) break;
/* With GICv5, the default PPI is what you get -- nothing else */ if (vgic_is_v5(vcpu->kvm) && irq != get_vgic_ppi(vcpu->kvm, default_ppi[i])) break;
/* * We know by construction that we only have PPIs, so all values * are less than 32 for non-GICv5 VGICs. On GICv5, they are * architecturally defined to be under 32 too. However, we mask * off most of the bits as we might be presented with a GICv5 * style PPI where the type is encoded in the top-bits. */ ppis |= BIT(irq & 0x1f); }
valid = hweight32(ppis) == nr_timers(vcpu);
if (valid) set_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, &vcpu->kvm->arch.flags);
mutex_unlock(&vcpu->kvm->arch.config_lock);
return valid;}
static bool kvm_arch_timer_get_input_level(int vintid){ struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
if (WARN(!vcpu, "No vcpu context!\n")) return false;
for (int i = 0; i < nr_timers(vcpu); i++) { struct arch_timer_context *ctx;
ctx = vcpu_get_timer(vcpu, i); if (timer_irq(ctx) == vintid) return kvm_timer_pending(ctx); }
/* A timer IRQ has fired, but no matching timer was found? */ WARN_RATELIMIT(1, "timer INTID%d unknown\n", vintid);
return false;}
int kvm_timer_enable(struct kvm_vcpu *vcpu){ struct arch_timer_cpu *timer = vcpu_timer(vcpu); const struct irq_ops *ops; struct timer_map map; int ret;
if (timer->enabled) return 0;
/* Without a VGIC we do not map virtual IRQs to physical IRQs */ if (!irqchip_in_kernel(vcpu->kvm)) goto no_vgic;
/* * At this stage, we have the guarantee that the vgic is both * available and initialized. */ if (!timer_irqs_are_valid(vcpu)) { kvm_debug("incorrectly configured timer irqs\n"); return -EINVAL; }
get_timer_map(vcpu, &map);
ops = vgic_is_v5(vcpu->kvm) ? &arch_timer_irq_ops_vgic_v5 : &arch_timer_irq_ops;
for (int i = 0; i < nr_timers(vcpu); i++) kvm_vgic_set_irq_ops(vcpu, timer_irq(vcpu_get_timer(vcpu, i)), ops);
ret = kvm_vgic_map_phys_irq(vcpu, map.direct_vtimer->host_timer_irq, timer_irq(map.direct_vtimer)); if (ret) return ret;
if (map.direct_ptimer) ret = kvm_vgic_map_phys_irq(vcpu, map.direct_ptimer->host_timer_irq, timer_irq(map.direct_ptimer)); if (ret) return ret;
no_vgic: timer->enabled = 1; return 0;}
/* If we have CNTPOFF, permanently set ECV to enable it */void kvm_timer_init_vhe(void){ if (cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF)) sysreg_clear_set(cnthctl_el2, 0, CNTHCTL_ECV);}
int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr){ int __user *uaddr = (int __user *)(long)attr->addr; int irq, idx, ret = 0;
if (!irqchip_in_kernel(vcpu->kvm)) return -EINVAL;
if (get_user(irq, uaddr)) return -EFAULT;
if (!(irq_is_ppi(vcpu->kvm, irq))) return -EINVAL;
guard(mutex)(&vcpu->kvm->arch.config_lock);
if (test_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, &vcpu->kvm->arch.flags)) { return -EBUSY; }
switch (attr->attr) { case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: idx = TIMER_VTIMER; break; case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: idx = TIMER_PTIMER; break; case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER: idx = TIMER_HVTIMER; break; case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER: idx = TIMER_HPTIMER; break; default: return -ENXIO; }
/* * We cannot validate the IRQ unicity before we run, so take it at * face value. The verdict will be given on first vcpu run, for each * vcpu. Yes this is late. Blame it on the stupid API. */ vcpu->kvm->arch.timer_data.ppi[idx] = irq;
return ret;}
int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr){ int __user *uaddr = (int __user *)(long)attr->addr; struct arch_timer_context *timer; int irq;
switch (attr->attr) { case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: timer = vcpu_vtimer(vcpu); break; case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: timer = vcpu_ptimer(vcpu); break; case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER: timer = vcpu_hvtimer(vcpu); break; case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER: timer = vcpu_hptimer(vcpu); break; default: return -ENXIO; }
irq = timer_irq(timer); return put_user(irq, uaddr);}
int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr){ switch (attr->attr) { case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER: case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER: return 0; }
return -ENXIO;}
int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm, struct kvm_arm_counter_offset *offset){ int ret = 0;
if (offset->reserved) return -EINVAL;
if (kvm_vm_is_protected(kvm)) return -EINVAL;
mutex_lock(&kvm->lock);
if (!kvm_trylock_all_vcpus(kvm)) { set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags);
/* * If userspace decides to set the offset using this * API rather than merely restoring the counter * values, the offset applies to both the virtual and * physical views. */ kvm->arch.timer_data.voffset = offset->counter_offset; kvm->arch.timer_data.poffset = offset->counter_offset;
kvm_unlock_all_vcpus(kvm); } else { ret = -EBUSY; }
mutex_unlock(&kvm->lock);
return ret;}