Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2017 SiFive */
#include <linux/acpi.h>#include <linux/of.h>#include <linux/prctl.h>#include <asm/acpi.h>#include <asm/cacheflush.h>
#ifdef CONFIG_SMP
#include <asm/sbi.h>
static void ipi_remote_fence_i(void *info){ return local_flush_icache_all();}
void flush_icache_all(void){ local_flush_icache_all();
if (num_online_cpus() < 2) return;
/* * Make sure all previous writes to the D$ are ordered before making * the IPI. The RISC-V spec states that a hart must execute a data fence * before triggering a remote fence.i in order to make the modification * visible for remote harts. * * IPIs on RISC-V are triggered by MMIO writes to either CLINT or * S-IMSIC, so the fence ensures previous data writes "happen before" * the MMIO. */ RISCV_FENCE(w, o);
if (riscv_use_sbi_for_rfence()) sbi_remote_fence_i(NULL); else on_each_cpu(ipi_remote_fence_i, NULL, 1);}EXPORT_SYMBOL(flush_icache_all);
/* * Performs an icache flush for the given MM context. RISC-V has no direct * mechanism for instruction cache shoot downs, so instead we send an IPI that * informs the remote harts they need to flush their local instruction caches. * To avoid pathologically slow behavior in a common case (a bunch of * single-hart processes on a many-hart machine, ie 'make -j') we avoid the * IPIs for harts that are not currently executing a MM context and instead * schedule a deferred local instruction cache flush to be performed before * execution resumes on each hart. */void flush_icache_mm(struct mm_struct *mm, bool local){ unsigned int cpu; cpumask_t others, *mask;
preempt_disable();
/* Mark every hart's icache as needing a flush for this MM. */ mask = &mm->context.icache_stale_mask; cpumask_setall(mask); /* Flush this hart's I$ now, and mark it as flushed. */ cpu = smp_processor_id(); cpumask_clear_cpu(cpu, mask); local_flush_icache_all();
/* * Flush the I$ of other harts concurrently executing, and mark them as * flushed. */ cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); local |= cpumask_empty(&others); if (mm == current->active_mm && local) { /* * It's assumed that at least one strongly ordered operation is * performed on this hart between setting a hart's cpumask bit * and scheduling this MM context on that hart. Sending an SBI * remote message will do this, but in the case where no * messages are sent we still need to order this hart's writes * with flush_icache_deferred(). */ smp_mb(); } else if (riscv_use_sbi_for_rfence()) { sbi_remote_fence_i(&others); } else { on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); }
preempt_enable();}
#endif /* CONFIG_SMP */
#ifdef CONFIG_MMUvoid flush_icache_pte(struct mm_struct *mm, pte_t pte){ struct folio *folio = page_folio(pte_page(pte));
if (!test_bit(PG_dcache_clean, &folio->flags.f)) { flush_icache_mm(mm, false); set_bit(PG_dcache_clean, &folio->flags.f); }}#endif /* CONFIG_MMU */
unsigned int riscv_cbom_block_size;EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
unsigned int riscv_cboz_block_size;EXPORT_SYMBOL_GPL(riscv_cboz_block_size);
unsigned int riscv_cbop_block_size;EXPORT_SYMBOL_GPL(riscv_cbop_block_size);
static void __init cbo_get_block_size(struct device_node *node, const char *name, u32 *block_size, unsigned long *first_hartid){ unsigned long hartid; u32 val;
if (riscv_of_processor_hartid(node, &hartid)) return;
if (of_property_read_u32(node, name, &val)) return;
if (!*block_size) { *block_size = val; *first_hartid = hartid; } else if (*block_size != val) { pr_warn("%s mismatched between harts %lu and %lu\n", name, *first_hartid, hartid); }}
void __init riscv_init_cbo_blocksizes(void){ unsigned long cbom_hartid, cboz_hartid, cbop_hartid; u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0; struct device_node *node; struct acpi_table_header *rhct; acpi_status status;
if (acpi_disabled) { for_each_of_cpu_node(node) { /* set block-size for cbom and/or cboz extension if available */ cbo_get_block_size(node, "riscv,cbom-block-size", &cbom_block_size, &cbom_hartid); cbo_get_block_size(node, "riscv,cboz-block-size", &cboz_block_size, &cboz_hartid); cbo_get_block_size(node, "riscv,cbop-block-size", &cbop_block_size, &cbop_hartid); } } else { status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); if (ACPI_FAILURE(status)) return;
acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size); acpi_put_table((struct acpi_table_header *)rhct); }
if (cbom_block_size) riscv_cbom_block_size = cbom_block_size;
if (cboz_block_size) riscv_cboz_block_size = cboz_block_size;
if (cbop_block_size) riscv_cbop_block_size = cbop_block_size;}
#ifdef CONFIG_SMPstatic void set_icache_stale_mask(void){ int cpu = get_cpu(); cpumask_t *mask; bool stale_cpu;
/* * Mark every other hart's icache as needing a flush for * this MM. Maintain the previous value of the current * cpu to handle the case when this function is called * concurrently on different harts. */ mask = ¤t->mm->context.icache_stale_mask; stale_cpu = cpumask_test_cpu(cpu, mask);
cpumask_setall(mask); __assign_cpu(cpu, mask, stale_cpu); put_cpu();}#endif
/** * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in * userspace. * @ctx: Set the type of icache flushing instructions permitted/prohibited in * userspace. Supported values described below. * * Supported values for ctx: * * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space. * * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``. * Therefore, caution must be taken; use this flag only when you can guarantee * that no thread in the process will emit fence.i from this point onward. * * @scope: Set scope of where icache flushing instructions are allowed to be * emitted. Supported values described below. * * Supported values for scope: * * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process * is coherent with instruction storage upon * migration. * * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is * coherent with instruction storage upon * migration. * * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are * permitted to emit icache flushing instructions. Whenever any thread in the * process is migrated, the corresponding hart's icache will be guaranteed to be * consistent with instruction storage. This does not enforce any guarantees * outside of migration. If a thread modifies an instruction that another thread * may attempt to execute, the other thread must still emit an icache flushing * instruction before attempting to execute the potentially modified * instruction. This must be performed by the user-space program. * * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the * thread calling this function is permitted to emit icache flushing * instructions. When the thread is migrated, the corresponding hart's icache * will be guaranteed to be consistent with instruction storage. * * On kernels configured without SMP, this function is a nop as migrations * across harts will not occur. */int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope){#ifdef CONFIG_SMP switch (ctx) { case PR_RISCV_CTX_SW_FENCEI_ON: switch (scope) { case PR_RISCV_SCOPE_PER_PROCESS: current->mm->context.force_icache_flush = true; break; case PR_RISCV_SCOPE_PER_THREAD: current->thread.force_icache_flush = true; break; default: return -EINVAL; } break; case PR_RISCV_CTX_SW_FENCEI_OFF: switch (scope) { case PR_RISCV_SCOPE_PER_PROCESS: set_icache_stale_mask(); current->mm->context.force_icache_flush = false; break; case PR_RISCV_SCOPE_PER_THREAD: set_icache_stale_mask(); current->thread.force_icache_flush = false; break; default: return -EINVAL; } break; default: return -EINVAL; } return 0;#else switch (ctx) { case PR_RISCV_CTX_SW_FENCEI_ON: case PR_RISCV_CTX_SW_FENCEI_OFF: return 0; default: return -EINVAL; }#endif}