Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/highmem.h>#include <linux/kprobes.h>
/** * flush_coherent_icache() - if a CPU has a coherent icache, flush it * Return true if the cache was flushed, false otherwise */static inline bool flush_coherent_icache(void){ /* * For a snooping icache, we still need a dummy icbi to purge all the * prefetched instructions from the ifetch buffers. We also need a sync * before the icbi to order the actual stores to memory that might * have modified instructions with the icbi. */ if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { mb(); /* sync */ icbi((void *)PAGE_OFFSET); mb(); /* sync */ isync(); return true; }
return false;}
/** * invalidate_icache_range() - Flush the icache by issuing icbi across an address range * @start: the start address * @stop: the stop address (exclusive) */static void invalidate_icache_range(unsigned long start, unsigned long stop){ unsigned long shift = l1_icache_shift(); unsigned long bytes = l1_icache_bytes(); char *addr = (char *)(start & ~(bytes - 1)); unsigned long size = stop - (unsigned long)addr + (bytes - 1); unsigned long i;
for (i = 0; i < size >> shift; i++, addr += bytes) icbi(addr);
mb(); /* sync */ isync();}
/** * flush_icache_range: Write any modified data cache blocks out to memory * and invalidate the corresponding blocks in the instruction cache * * Generic code will call this after writing memory, before executing from it. * * @start: the start address * @stop: the stop address (exclusive) */void flush_icache_range(unsigned long start, unsigned long stop){ if (flush_coherent_icache()) return;
clean_dcache_range(start, stop);
if (IS_ENABLED(CONFIG_44x)) { /* * Flash invalidate on 44x because we are passed kmapped * addresses and this doesn't work for userspace pages due to * the virtually tagged icache. */ iccci((void *)start); mb(); /* sync */ isync(); } else invalidate_icache_range(start, stop);}EXPORT_SYMBOL(flush_icache_range);
#ifdef CONFIG_HIGHMEM/** * flush_dcache_icache_phys() - Flush a page by its physical address * @physaddr: the physical address of the page */static void flush_dcache_icache_phys(unsigned long physaddr){ unsigned long bytes = l1_dcache_bytes(); unsigned long nb = PAGE_SIZE / bytes; unsigned long addr = physaddr & PAGE_MASK; unsigned long msr, msr0; unsigned long loop1 = addr, loop2 = addr;
msr0 = mfmsr(); msr = msr0 & ~MSR_DR; /* * This must remain as ASM to prevent potential memory accesses * while the data MMU is disabled */ asm volatile( " mtctr %2;\n" " mtmsr %3;\n" " isync;\n" "0: dcbst 0, %0;\n" " addi %0, %0, %4;\n" " bdnz 0b;\n" " sync;\n" " mtctr %2;\n" "1: icbi 0, %1;\n" " addi %1, %1, %4;\n" " bdnz 1b;\n" " sync;\n" " mtmsr %5;\n" " isync;\n" : "+&r" (loop1), "+&r" (loop2) : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0) : "ctr", "memory");}NOKPROBE_SYMBOL(flush_dcache_icache_phys)#elsestatic void flush_dcache_icache_phys(unsigned long physaddr){}#endif
/** * __flush_dcache_icache(): Flush a particular page from the data cache to RAM. * Note: this is necessary because the instruction cache does *not* * snoop from the data cache. * * @p: the address of the page to flush */static void __flush_dcache_icache(void *p){ unsigned long addr = (unsigned long)p & PAGE_MASK;
clean_dcache_range(addr, addr + PAGE_SIZE);
/* * We don't flush the icache on 44x. Those have a virtual icache and we * don't have access to the virtual address here (it's not the page * vaddr but where it's mapped in user space). The flushing of the * icache on these is handled elsewhere, when a change in the address * space occurs, before returning to user space. */
if (mmu_has_feature(MMU_FTR_TYPE_44x)) return;
invalidate_icache_range(addr, addr + PAGE_SIZE);}
void flush_dcache_icache_folio(struct folio *folio){ unsigned int i, nr = folio_nr_pages(folio);
if (flush_coherent_icache()) return;
if (!folio_test_highmem(folio)) { void *addr = folio_address(folio); for (i = 0; i < nr; i++) __flush_dcache_icache(addr + i * PAGE_SIZE); } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { for (i = 0; i < nr; i++) { void *start = kmap_local_folio(folio, i * PAGE_SIZE);
__flush_dcache_icache(start); kunmap_local(start); } } else { unsigned long pfn = folio_pfn(folio); for (i = 0; i < nr; i++) flush_dcache_icache_phys((pfn + i) * PAGE_SIZE); }}EXPORT_SYMBOL(flush_dcache_icache_folio);
void clear_user_page(void *page, unsigned long vaddr, struct page *pg){ clear_page(page);
/* * We shouldn't have to do this, but some versions of glibc * require it (ld.so assumes zero filled pages are icache clean) * - Anton */ flush_dcache_page(pg);}EXPORT_SYMBOL(clear_user_page);
void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, struct page *pg){ copy_page(vto, vfrom);
/* * We should be able to use the following optimisation, however * there are two problems. * Firstly a bug in some versions of binutils meant PLT sections * were not marked executable. * Secondly the first word in the GOT section is blrl, used * to establish the GOT address. Until recently the GOT was * not marked executable. * - Anton */#if 0 if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0)) return;#endif
flush_dcache_page(pg);}
void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, unsigned long addr, int len){ void *maddr;
maddr = kmap_local_page(page) + (addr & ~PAGE_MASK); flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len); kunmap_local(maddr);}