Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2015 Imagination Technologies Ltd * Author: Qais Yousef <qais.yousef@imgtec.com> * * This file contains driver APIs to the IPI subsystem. */
#define pr_fmt(fmt) "genirq/ipi: " fmt
#include <linux/irqdomain.h>#include <linux/irq.h>
/** * irq_reserve_ipi() - Setup an IPI to destination cpumask * @domain: IPI domain * @dest: cpumask of CPUs which can receive the IPI * * Allocate a virq that can be used to send IPI to any CPU in dest mask. * * Return: Linux IRQ number on success or error code on failure */int irq_reserve_ipi(struct irq_domain *domain, const struct cpumask *dest){ unsigned int nr_irqs, offset; struct irq_data *data; int virq, i;
if (!domain ||!irq_domain_is_ipi(domain)) { pr_warn("Reservation on a non IPI domain\n"); return -EINVAL; }
if (!cpumask_subset(dest, cpu_possible_mask)) { pr_warn("Reservation is not in possible_cpu_mask\n"); return -EINVAL; }
nr_irqs = cpumask_weight(dest); if (!nr_irqs) { pr_warn("Reservation for empty destination mask\n"); return -EINVAL; }
if (irq_domain_is_ipi_single(domain)) { /* * If the underlying implementation uses a single HW irq on * all cpus then we only need a single Linux irq number for * it. We have no restrictions vs. the destination mask. The * underlying implementation can deal with holes nicely. */ nr_irqs = 1; offset = 0; } else { unsigned int next;
/* * The IPI requires a separate HW irq on each CPU. We require * that the destination mask is consecutive. If an * implementation needs to support holes, it can reserve * several IPI ranges. */ offset = cpumask_first(dest); /* * Find a hole and if found look for another set bit after the * hole. For now we don't support this scenario. */ next = cpumask_next_zero(offset, dest); if (next < nr_cpu_ids) next = cpumask_next(next, dest); if (next < nr_cpu_ids) { pr_warn("Destination mask has holes\n"); return -EINVAL; } }
virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL); if (virq <= 0) { pr_warn("Can't reserve IPI, failed to alloc descs\n"); return -ENOMEM; }
virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, (void *) dest, true, NULL);
if (virq <= 0) { pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); goto free_descs; }
for (i = 0; i < nr_irqs; i++) { data = irq_get_irq_data(virq + i); cpumask_copy(data->common->affinity, dest); data->common->ipi_offset = offset; irq_set_status_flags(virq + i, IRQ_NO_BALANCING); } return virq;
free_descs: irq_free_descs(virq, nr_irqs); return -EBUSY;}
/** * irq_destroy_ipi() - unreserve an IPI that was previously allocated * @irq: Linux IRQ number to be destroyed * @dest: cpumask of CPUs which should have the IPI removed * * The IPIs allocated with irq_reserve_ipi() are returned to the system * destroying all virqs associated with them. * * Return: %0 on success or error code on failure. */int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest){ struct irq_data *data = irq_get_irq_data(irq); const struct cpumask *ipimask; struct irq_domain *domain; unsigned int nr_irqs;
if (!irq || !data) return -EINVAL;
domain = data->domain; if (WARN_ON(domain == NULL)) return -EINVAL;
if (!irq_domain_is_ipi(domain)) { pr_warn("Trying to destroy a non IPI domain!\n"); return -EINVAL; }
ipimask = irq_data_get_affinity_mask(data); if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask))) /* * Must be destroying a subset of CPUs to which this IPI * was set up to target */ return -EINVAL;
if (irq_domain_is_ipi_per_cpu(domain)) { irq = irq + cpumask_first(dest) - data->common->ipi_offset; nr_irqs = cpumask_weight(dest); } else { nr_irqs = 1; }
irq_domain_free_irqs(irq, nr_irqs); return 0;}
/** * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU * @irq: Linux IRQ number * @cpu: the target CPU * * When dealing with coprocessors IPI, we need to inform the coprocessor of * the hwirq it needs to use to receive and send IPIs. * * Return: hwirq value on success or INVALID_HWIRQ on failure. */irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu){ struct irq_data *data = irq_get_irq_data(irq); const struct cpumask *ipimask;
if (!data || cpu >= nr_cpu_ids) return INVALID_HWIRQ;
ipimask = irq_data_get_affinity_mask(data); if (!ipimask || !cpumask_test_cpu(cpu, ipimask)) return INVALID_HWIRQ;
/* * Get the real hardware irq number if the underlying implementation * uses a separate irq per cpu. If the underlying implementation uses * a single hardware irq for all cpus then the IPI send mechanism * needs to take care of the cpu destinations. */ if (irq_domain_is_ipi_per_cpu(data->domain)) data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;}EXPORT_SYMBOL_GPL(ipi_get_hwirq);
static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, const struct cpumask *dest, unsigned int cpu){ const struct cpumask *ipimask;
if (!chip || !data) return -EINVAL;
if (!chip->ipi_send_single && !chip->ipi_send_mask) return -EINVAL;
if (cpu >= nr_cpu_ids) return -EINVAL;
ipimask = irq_data_get_affinity_mask(data); if (!ipimask) return -EINVAL;
if (dest) { if (!cpumask_subset(dest, ipimask)) return -EINVAL; } else { if (!cpumask_test_cpu(cpu, ipimask)) return -EINVAL; } return 0;}
/** * __ipi_send_single - send an IPI to a target Linux SMP CPU * @desc: pointer to irq_desc of the IRQ * @cpu: destination CPU, must in the destination mask passed to * irq_reserve_ipi() * * This function is for architecture or core code to speed up IPI sending. Not * usable from driver code. * * Return: %0 on success or negative error number on failure. */int __ipi_send_single(struct irq_desc *desc, unsigned int cpu){ struct irq_data *data = irq_desc_get_irq_data(desc); struct irq_chip *chip = irq_data_get_irq_chip(data);
#ifdef DEBUG /* * Minimise the overhead by omitting the checks for Linux SMP IPIs. * Since the callers should be arch or core code which is generally * trusted, only check for errors when debugging. */ if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) return -EINVAL;#endif if (!chip->ipi_send_single) { chip->ipi_send_mask(data, cpumask_of(cpu)); return 0; }
/* FIXME: Store this information in irqdata flags */ if (irq_domain_is_ipi_per_cpu(data->domain) && cpu != data->common->ipi_offset) { /* use the correct data for that cpu */ unsigned irq = data->irq + cpu - data->common->ipi_offset;
data = irq_get_irq_data(irq); } chip->ipi_send_single(data, cpu); return 0;}
/** * __ipi_send_mask - send an IPI to target Linux SMP CPU(s) * @desc: pointer to irq_desc of the IRQ * @dest: dest CPU(s), must be a subset of the mask passed to * irq_reserve_ipi() * * This function is for architecture or core code to speed up IPI sending. Not * usable from driver code. * * Return: %0 on success or negative error number on failure. */int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest){ struct irq_data *data = irq_desc_get_irq_data(desc); struct irq_chip *chip = irq_data_get_irq_chip(data); unsigned int cpu;
#ifdef DEBUG /* * Minimise the overhead by omitting the checks for Linux SMP IPIs. * Since the callers should be arch or core code which is generally * trusted, only check for errors when debugging. */ if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) return -EINVAL;#endif if (chip->ipi_send_mask) { chip->ipi_send_mask(data, dest); return 0; }
if (irq_domain_is_ipi_per_cpu(data->domain)) { unsigned int base = data->irq;
for_each_cpu(cpu, dest) { unsigned irq = base + cpu - data->common->ipi_offset;
data = irq_get_irq_data(irq); chip->ipi_send_single(data, cpu); } } else { for_each_cpu(cpu, dest) chip->ipi_send_single(data, cpu); } return 0;}
/** * ipi_send_single - Send an IPI to a single CPU * @virq: Linux IRQ number from irq_reserve_ipi() * @cpu: destination CPU, must in the destination mask passed to * irq_reserve_ipi() * * Return: %0 on success or negative error number on failure. */int ipi_send_single(unsigned int virq, unsigned int cpu){ struct irq_desc *desc = irq_to_desc(virq); struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) return -EINVAL;
return __ipi_send_single(desc, cpu);}EXPORT_SYMBOL_GPL(ipi_send_single);
/** * ipi_send_mask - Send an IPI to target CPU(s) * @virq: Linux IRQ number from irq_reserve_ipi() * @dest: dest CPU(s), must be a subset of the mask passed to * irq_reserve_ipi() * * Return: %0 on success or negative error number on failure. */int ipi_send_mask(unsigned int virq, const struct cpumask *dest){ struct irq_desc *desc = irq_to_desc(virq); struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) return -EINVAL;
return __ipi_send_mask(desc, dest);}EXPORT_SYMBOL_GPL(ipi_send_mask);