Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar * Copyright (C) 2005-2006, Thomas Gleixner, Russell King * * This file contains the core interrupt handling code. Detailed * information is available in Documentation/core-api/genericirq.rst * */
#include <linux/irq.h>#include <linux/random.h>#include <linux/sched.h>#include <linux/interrupt.h>#include <linux/kernel_stat.h>
#include <asm/irq_regs.h>
#include <trace/events/irq.h>
#include "internals.h"
#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLERvoid (*handle_arch_irq)(struct pt_regs *) __ro_after_init;#endif
/** * handle_bad_irq - handle spurious and unhandled irqs * @desc: description of the interrupt * * Handles spurious and unhandled IRQ's. It also prints a debugmessage. */void handle_bad_irq(struct irq_desc *desc){ unsigned int irq = irq_desc_get_irq(desc);
print_irq_desc(irq, desc); kstat_incr_irqs_this_cpu(desc); ack_bad_irq(irq);}EXPORT_SYMBOL_GPL(handle_bad_irq);
/* * Special, empty irq handler: */irqreturn_t no_action(int cpl, void *dev_id){ return IRQ_NONE;}EXPORT_SYMBOL_GPL(no_action);
static void warn_no_thread(unsigned int irq, struct irqaction *action){ if (test_and_set_bit(IRQTF_WARNED, &action->thread_flags)) return;
printk(KERN_WARNING "IRQ %d device %s returned IRQ_WAKE_THREAD " "but no thread function available.", irq, action->name);}
void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action){ /* * In case the thread crashed and was killed we just pretend that * we handled the interrupt. The hardirq handler has disabled the * device interrupt, so no irq storm is lurking. */ if (action->thread->flags & PF_EXITING) return;
/* * Wake up the handler thread for this action. If the * RUNTHREAD bit is already set, nothing to do. */ if (test_and_set_bit(IRQTF_RUNTHREAD, &action->thread_flags)) return;
/* * It's safe to OR the mask lockless here. We have only two * places which write to threads_oneshot: This code and the * irq thread. * * This code is the hard irq context and can never run on two * cpus in parallel. If it ever does we have more serious * problems than this bitmask. * * The irq threads of this irq which clear their "running" bit * in threads_oneshot are serialized via desc->lock against * each other and they are serialized against this code by * IRQS_INPROGRESS. * * Hard irq handler: * * spin_lock(desc->lock); * desc->state |= IRQS_INPROGRESS; * spin_unlock(desc->lock); * set_bit(IRQTF_RUNTHREAD, &action->thread_flags); * desc->threads_oneshot |= mask; * spin_lock(desc->lock); * desc->state &= ~IRQS_INPROGRESS; * spin_unlock(desc->lock); * * irq thread: * * again: * spin_lock(desc->lock); * if (desc->state & IRQS_INPROGRESS) { * spin_unlock(desc->lock); * while(desc->state & IRQS_INPROGRESS) * cpu_relax(); * goto again; * } * if (!test_bit(IRQTF_RUNTHREAD, &action->thread_flags)) * desc->threads_oneshot &= ~mask; * spin_unlock(desc->lock); * * So either the thread waits for us to clear IRQS_INPROGRESS * or we are waiting in the flow handler for desc->lock to be * released before we reach this point. The thread also checks * IRQTF_RUNTHREAD under desc->lock. If set it leaves * threads_oneshot untouched and runs the thread another time. */ desc->threads_oneshot |= action->thread_mask;
/* * We increment the threads_active counter in case we wake up * the irq thread. The irq thread decrements the counter when * it returns from the handler or in the exit path and wakes * up waiters which are stuck in synchronize_irq() when the * active count becomes zero. synchronize_irq() is serialized * against this code (hard irq handler) via IRQS_INPROGRESS * like the finalize_oneshot() code. See comment above. */ atomic_inc(&desc->threads_active);
/* * This might be a premature wakeup before the thread reached the * thread function and set the IRQTF_READY bit. It's waiting in * kthread code with state UNINTERRUPTIBLE. Once it reaches the * thread function it waits with INTERRUPTIBLE. The wakeup is not * lost in that case because the thread is guaranteed to observe * the RUN flag before it goes to sleep in wait_for_interrupt(). */ wake_up_state(action->thread, TASK_INTERRUPTIBLE);}
static DEFINE_STATIC_KEY_FALSE(irqhandler_duration_check_enabled);static u64 irqhandler_duration_threshold_ns __ro_after_init;
static int __init irqhandler_duration_check_setup(char *arg){ unsigned long val; int ret;
ret = kstrtoul(arg, 0, &val); if (ret) { pr_err("Unable to parse irqhandler.duration_warn_us setting: ret=%d\n", ret); return 0; }
if (!val) { pr_err("Invalid irqhandler.duration_warn_us setting, must be > 0\n"); return 0; }
irqhandler_duration_threshold_ns = val * 1000; static_branch_enable(&irqhandler_duration_check_enabled);
return 1;}__setup("irqhandler.duration_warn_us=", irqhandler_duration_check_setup);
static inline void irqhandler_duration_check(u64 ts_start, unsigned int irq, const struct irqaction *action){ u64 delta_ns = local_clock() - ts_start;
if (unlikely(delta_ns > irqhandler_duration_threshold_ns)) { pr_warn_ratelimited("[CPU%u] long duration of IRQ[%u:%ps], took: %llu us\n", smp_processor_id(), irq, action->handler, div_u64(delta_ns, NSEC_PER_USEC)); }}
irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc){ irqreturn_t retval = IRQ_NONE; unsigned int irq = desc->irq_data.irq; struct irqaction *action;
for_each_action_of_desc(desc, action) { irqreturn_t res;
/* * If this IRQ would be threaded under force_irqthreads, mark it so. */ if (irq_settings_can_thread(desc) && !(action->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))) lockdep_hardirq_threaded();
trace_irq_handler_entry(irq, action);
if (static_branch_unlikely(&irqhandler_duration_check_enabled)) { u64 ts_start = local_clock();
res = action->handler(irq, action->dev_id); irqhandler_duration_check(ts_start, irq, action); } else { res = action->handler(irq, action->dev_id); }
trace_irq_handler_exit(irq, action, res);
if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pS enabled interrupts\n", irq, action->handler)) local_irq_disable();
switch (res) { case IRQ_WAKE_THREAD: /* * Catch drivers which return WAKE_THREAD but * did not set up a thread function */ if (unlikely(!action->thread_fn)) { warn_no_thread(irq, action); break; }
__irq_wake_thread(desc, action); break;
default: break; }
retval |= res; }
return retval;}
irqreturn_t handle_irq_event_percpu(struct irq_desc *desc){ irqreturn_t retval;
retval = __handle_irq_event_percpu(desc);
add_interrupt_randomness(desc->irq_data.irq);
if (!irq_settings_no_debug(desc)) note_interrupt(desc, retval); return retval;}
irqreturn_t handle_irq_event(struct irq_desc *desc){ irqreturn_t ret;
desc->istate &= ~IRQS_PENDING; irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS); raw_spin_unlock(&desc->lock);
ret = handle_irq_event_percpu(desc);
raw_spin_lock(&desc->lock); irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS); return ret;}
#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLERint __init set_handle_irq(void (*handle_irq)(struct pt_regs *)){ if (handle_arch_irq) return -EBUSY;
handle_arch_irq = handle_irq; return 0;}
/** * generic_handle_arch_irq - root irq handler for architectures which do no * entry accounting themselves * @regs: Register file coming from the low-level handling code */asmlinkage void noinstr generic_handle_arch_irq(struct pt_regs *regs){ struct pt_regs *old_regs;
irq_enter(); old_regs = set_irq_regs(regs); handle_arch_irq(regs); set_irq_regs(old_regs); irq_exit();}#endif