Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_PGTABLE_H#define _LINUX_PGTABLE_H
#include <linux/pfn.h>#include <asm/pgtable.h>
#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)#define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT)
#ifndef __ASSEMBLY__#ifdef CONFIG_MMU
#include <linux/mm_types.h>#include <linux/bug.h>#include <linux/errno.h>#include <asm-generic/pgtable_uffd.h>#include <linux/page_table_check.h>
#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED#endif
/* * This defines the generic helper for accessing PMD page * table page. Although platforms can still override this * via their respective <asm/pgtable.h>. */#ifndef pmd_pgtable#define pmd_pgtable(pmd) pmd_page(pmd)#endif
#define pmd_folio(pmd) page_folio(pmd_page(pmd))
/* * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] * * The pXx_index() functions return the index of the entry in the page * table page which would control the given virtual address * * As these functions may be used by the same code for different levels of * the page table folding, they are always available, regardless of * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 * because in such cases PTRS_PER_PxD equals 1. */
static inline unsigned long pte_index(unsigned long address){ return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);}
#ifndef pmd_indexstatic inline unsigned long pmd_index(unsigned long address){ return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);}#define pmd_index pmd_index#endif
#ifndef pud_indexstatic inline unsigned long pud_index(unsigned long address){ return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);}#define pud_index pud_index#endif
#ifndef pgd_index/* Must be a compile-time constant, so implement it as a macro */#define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))#endif
#ifndef kernel_pte_initstatic inline void kernel_pte_init(void *addr){}#define kernel_pte_init kernel_pte_init#endif
#ifndef pmd_initstatic inline void pmd_init(void *addr){}#define pmd_init pmd_init#endif
#ifndef pud_initstatic inline void pud_init(void *addr){}#define pud_init pud_init#endif
#ifndef pte_offset_kernelstatic inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address){ return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);}#define pte_offset_kernel pte_offset_kernel#endif
#ifdef CONFIG_HIGHPTE#define __pte_map(pmd, address) \ ((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address)))#define pte_unmap(pte) do { \ kunmap_local((pte)); \ rcu_read_unlock(); \} while (0)#elsestatic inline pte_t *__pte_map(pmd_t *pmd, unsigned long address){ return pte_offset_kernel(pmd, address);}static inline void pte_unmap(pte_t *pte){ rcu_read_unlock();}#endif
void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable);
/* Find an entry in the second-level page table.. */#ifndef pmd_offsetstatic inline pmd_t *pmd_offset(pud_t *pud, unsigned long address){ return pud_pgtable(*pud) + pmd_index(address);}#define pmd_offset pmd_offset#endif
#ifndef pud_offsetstatic inline pud_t *pud_offset(p4d_t *p4d, unsigned long address){ return p4d_pgtable(*p4d) + pud_index(address);}#define pud_offset pud_offset#endif
static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address){ return (pgd + pgd_index(address));};
/* * a shortcut to get a pgd_t in a given mm */#ifndef pgd_offset#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))#endif
/* * a shortcut which implies the use of the kernel's pgd, instead * of a process's */#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
/* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a * pointer to the PMD entry in user or kernel page table or translate a virtual * address to the pointer in the PTE in the kernel page tables with simple * helpers. */static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va){ return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va);}
static inline pmd_t *pmd_off_k(unsigned long va){ return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va);}
static inline pte_t *virt_to_kpte(unsigned long vaddr){ pmd_t *pmd = pmd_off_k(vaddr);
return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);}
#ifndef pmd_youngstatic inline int pmd_young(pmd_t pmd){ return 0;}#endif
#ifndef pmd_dirtystatic inline int pmd_dirty(pmd_t pmd){ return 0;}#endif
/* * A facility to provide lazy MMU batching. This allows PTE updates and * page invalidations to be delayed until a call to leave lazy MMU mode * is issued. Some architectures may benefit from doing this, and it is * beneficial for both shadow and direct mode hypervisors, which may batch * the PTE updates which happen during this window. Note that using this * interface requires that read hazards be removed from the code. A read * hazard could result in the direct mode hypervisor case, since the actual * write to the page tables may not yet have taken place, so reads though * a raw PTE pointer after it has been modified are not guaranteed to be * up to date. * * In the general case, no lock is guaranteed to be held between entry and exit * of the lazy mode. (In practice, for user PTE updates, the appropriate page * table lock(s) are held, but for kernel PTE updates, no lock is held). * The implementation must therefore assume preemption may be enabled upon * entry to the mode and cpu migration is possible; it must take steps to be * robust against this. An implementation may handle this by disabling * preemption, as a consequence generic code may not sleep while the lazy MMU * mode is active. * * The mode is disabled in interrupt context and calls to the lazy_mmu API have * no effect. * * The lazy MMU mode is enabled for a given block of code using: * * lazy_mmu_mode_enable(); * <code> * lazy_mmu_mode_disable(); * * Nesting is permitted: <code> may itself use an enable()/disable() pair. * A nested call to enable() has no functional effect; however disable() causes * any batched architectural state to be flushed regardless of nesting. After a * call to disable(), the caller can therefore rely on all previous page table * modifications to have taken effect, but the lazy MMU mode may still be * enabled. * * In certain cases, it may be desirable to temporarily pause the lazy MMU mode. * This can be done using: * * lazy_mmu_mode_pause(); * <code> * lazy_mmu_mode_resume(); * * pause() ensures that the mode is exited regardless of the nesting level; * resume() re-enters the mode at the same nesting level. Any call to the * lazy_mmu_mode_* API between those two calls has no effect. In particular, * this means that pause()/resume() pairs may nest. * * is_lazy_mmu_mode_active() can be used to check whether the lazy MMU mode is * currently enabled. */#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE/** * lazy_mmu_mode_enable() - Enable the lazy MMU mode. * * Enters a new lazy MMU mode section; if the mode was not already enabled, * enables it and calls arch_enter_lazy_mmu_mode(). * * Must be paired with a call to lazy_mmu_mode_disable(). * * Has no effect if called: * - While paused - see lazy_mmu_mode_pause() * - In interrupt context */static inline void lazy_mmu_mode_enable(void){ struct lazy_mmu_state *state = ¤t->lazy_mmu_state;
if (in_interrupt() || state->pause_count > 0) return;
VM_WARN_ON_ONCE(state->enable_count == U8_MAX);
if (state->enable_count++ == 0) arch_enter_lazy_mmu_mode();}
/** * lazy_mmu_mode_disable() - Disable the lazy MMU mode. * * Exits the current lazy MMU mode section. If it is the outermost section, * disables the mode and calls arch_leave_lazy_mmu_mode(). Otherwise (nested * section), calls arch_flush_lazy_mmu_mode(). * * Must match a call to lazy_mmu_mode_enable(). * * Has no effect if called: * - While paused - see lazy_mmu_mode_pause() * - In interrupt context */static inline void lazy_mmu_mode_disable(void){ struct lazy_mmu_state *state = ¤t->lazy_mmu_state;
if (in_interrupt() || state->pause_count > 0) return;
VM_WARN_ON_ONCE(state->enable_count == 0);
if (--state->enable_count == 0) arch_leave_lazy_mmu_mode(); else /* Exiting a nested section */ arch_flush_lazy_mmu_mode();
}
/** * __task_lazy_mmu_mode_pause() - Pause the lazy MMU mode for a task. * @tsk: The task to check. * * Pauses the lazy MMU mode of @tsk. * * This function only operates on the state saved in task_struct; to pause * current lazy_mmu_mode_pause() should be used instead. * * This function is intended for architectures that implement the lazy MMU * mode; it must not be called from generic code. */static inline void __task_lazy_mmu_mode_pause(struct task_struct *tsk){ struct lazy_mmu_state *state = &tsk->lazy_mmu_state;
VM_WARN_ON_ONCE(state->pause_count == U8_MAX);
if (state->pause_count++ == 0 && state->enable_count > 0) arch_leave_lazy_mmu_mode();}
/** * lazy_mmu_mode_pause() - Pause the lazy MMU mode. * * Pauses the lazy MMU mode; if it is currently active, disables it and calls * arch_leave_lazy_mmu_mode(). * * Must be paired with a call to lazy_mmu_mode_resume(). Calls to the * lazy_mmu_mode_* API have no effect until the matching resume() call. * * Has no effect if called: * - While paused (inside another pause()/resume() pair) * - In interrupt context */static inline void lazy_mmu_mode_pause(void){ if (in_interrupt()) return;
__task_lazy_mmu_mode_pause(current);}
/** * __task_lazy_mmu_mode_resume() - Resume the lazy MMU mode for a task. * @tsk: The task to check. * * Resumes the lazy MMU mode of @tsk. * * This function only operates on the state saved in task_struct; to resume * current lazy_mmu_mode_resume() should be used instead. * * This function is intended for architectures that implement the lazy MMU * mode; it must not be called from generic code. */static inline void __task_lazy_mmu_mode_resume(struct task_struct *tsk){ struct lazy_mmu_state *state = &tsk->lazy_mmu_state;
VM_WARN_ON_ONCE(state->pause_count == 0);
if (--state->pause_count == 0 && state->enable_count > 0) arch_enter_lazy_mmu_mode();}
/** * lazy_mmu_mode_resume() - Resume the lazy MMU mode. * * Resumes the lazy MMU mode; if it was active at the point where the matching * call to lazy_mmu_mode_pause() was made, re-enables it and calls * arch_enter_lazy_mmu_mode(). * * Must match a call to lazy_mmu_mode_pause(). * * Has no effect if called: * - While paused (inside another pause()/resume() pair) * - In interrupt context */static inline void lazy_mmu_mode_resume(void){ if (in_interrupt()) return;
__task_lazy_mmu_mode_resume(current);}#elsestatic inline void lazy_mmu_mode_enable(void) {}static inline void lazy_mmu_mode_disable(void) {}static inline void lazy_mmu_mode_pause(void) {}static inline void lazy_mmu_mode_resume(void) {}#endif
#ifndef pte_batch_hint/** * pte_batch_hint - Number of pages that can be added to batch without scanning. * @ptep: Page table pointer for the entry. * @pte: Page table entry. * * Some architectures know that a set of contiguous ptes all map the same * contiguous memory with the same permissions. In this case, it can provide a * hint to aid pte batching without the core code needing to scan every pte. * * An architecture implementation may ignore the PTE accessed state. Further, * the dirty state must apply atomically to all the PTEs described by the hint. * * May be overridden by the architecture, else pte_batch_hint is always 1. */static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte){ return 1;}#endif
#ifndef pte_advance_pfnstatic inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr){ return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT));}#endif
#define pte_next_pfn(pte) pte_advance_pfn(pte, 1)
#ifndef set_ptes/** * set_ptes - Map consecutive pages to a contiguous range of addresses. * @mm: Address space to map the pages into. * @addr: Address to map the first page at. * @ptep: Page table pointer for the first entry. * @pte: Page table entry for the first page. * @nr: Number of pages to map. * * When nr==1, initial state of pte may be present or not present, and new state * may be present or not present. When nr>1, initial state of all ptes must be * not present, and new state must be present. * * May be overridden by the architecture, or the architecture can define * set_pte() and PFN_PTE_SHIFT. * * Context: The caller holds the page table lock. The pages all belong * to the same folio. The PTEs are all in the same PMD. */static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr){ page_table_check_ptes_set(mm, addr, ptep, pte, nr);
for (;;) { set_pte(ptep, pte); if (--nr == 0) break; ptep++; pte = pte_next_pfn(pte); }}#endif#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1)
#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGSextern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty);#endif
#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS#ifdef CONFIG_TRANSPARENT_HUGEPAGEextern int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty);extern int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t entry, int dirty);#elsestatic inline int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty){ BUILD_BUG(); return 0;}static inline int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t entry, int dirty){ BUILD_BUG(); return 0;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif
#ifndef ptep_getstatic inline pte_t ptep_get(pte_t *ptep){ return READ_ONCE(*ptep);}#endif
#ifndef pmdp_getstatic inline pmd_t pmdp_get(pmd_t *pmdp){ return READ_ONCE(*pmdp);}#endif
#ifndef pudp_getstatic inline pud_t pudp_get(pud_t *pudp){ return READ_ONCE(*pudp);}#endif
#ifndef p4dp_getstatic inline p4d_t p4dp_get(p4d_t *p4dp){ return READ_ONCE(*p4dp);}#endif
#ifndef pgdp_getstatic inline pgd_t pgdp_get(pgd_t *pgdp){ return READ_ONCE(*pgdp);}#endif
#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNGstatic inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep){ pte_t pte = ptep_get(ptep); bool young = true;
if (!pte_young(pte)) young = false; else set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); return young;}#endif
#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ pmd_t pmd = *pmdp; bool young = true;
if (!pmd_young(pmd)) young = false; else set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); return young;}#elsestatic inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ BUILD_BUG(); return false;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSHbool ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep);#endif
#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH#ifdef CONFIG_TRANSPARENT_HUGEPAGEbool pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#else/* * Despite relevant to THP only, this API is called from generic rmap code * under PageTransHuge(), hence needs a dummy implementation for !THP */static inline bool pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ BUILD_BUG(); return false;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif
#ifndef arch_has_hw_nonleaf_pmd_young/* * Return whether the accessed bit in non-leaf PMD entries is supported on the * local CPU. */static inline bool arch_has_hw_nonleaf_pmd_young(void){ return IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG);}#endif
#ifndef arch_has_hw_pte_young/* * Return whether the accessed bit is supported on the local CPU. * * This stub assumes accessing through an old PTE triggers a page fault. * Architectures that automatically set the access bit should overwrite it. */static inline bool arch_has_hw_pte_young(void){ return IS_ENABLED(CONFIG_ARCH_HAS_HW_PTE_YOUNG);}#endif
#ifndef exec_folio_order/* * Returns preferred minimum folio order for executable file-backed memory. Must * be in range [0, PMD_ORDER). Default to order-0. */static inline unsigned int exec_folio_order(void){ return 0;}#endif
#ifndef arch_check_zapped_ptestatic inline void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte){}#endif
#ifndef arch_check_zapped_pmdstatic inline void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd){}#endif
#ifndef arch_check_zapped_pudstatic inline void arch_check_zapped_pud(struct vm_area_struct *vma, pud_t pud){}#endif
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEARstatic inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, pte_t *ptep){ pte_t pte = ptep_get(ptep); pte_clear(mm, address, ptep); page_table_check_pte_clear(mm, address, pte); return pte;}#endif
#ifndef clear_young_dirty_ptes/** * clear_young_dirty_ptes - Mark PTEs that map consecutive pages of the * same folio as old/clean. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to mark old/clean. * @flags: Flags to modify the PTE batch semantics. * * May be overridden by the architecture; otherwise, implemented by * get_and_clear/modify/set for each pte in the range. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags){ pte_t pte;
for (;;) { if (flags == CYDP_CLEAR_YOUNG) ptep_test_and_clear_young(vma, addr, ptep); else { pte = ptep_get_and_clear(vma->vm_mm, addr, ptep); if (flags & CYDP_CLEAR_YOUNG) pte = pte_mkold(pte); if (flags & CYDP_CLEAR_DIRTY) pte = pte_mkclean(pte); set_pte_at(vma->vm_mm, addr, ptep, pte); } if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep){ pte_t pte = ptep_get(ptep);
pte_clear(mm, addr, ptep); /* * No need for ptep_get_and_clear(): page table check doesn't care about * any bits that could have been set by HW concurrently. */ page_table_check_pte_clear(mm, addr, pte);}
#ifdef CONFIG_GUP_GET_PXX_LOW_HIGH/* * For walking the pagetables without holding any locks. Some architectures * (eg x86-32 PAE) cannot load the entries atomically without using expensive * instructions. We are guaranteed that a PTE will only either go from not * present to present, or present to not present -- it will not switch to a * completely different present page without a TLB flush inbetween; which we * are blocking by holding interrupts off. * * Setting ptes from not present to present goes: * * ptep->pte_high = h; * smp_wmb(); * ptep->pte_low = l; * * And present to not present goes: * * ptep->pte_low = 0; * smp_wmb(); * ptep->pte_high = 0; * * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'. * We load pte_high *after* loading pte_low, which ensures we don't see an older * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't * picked up a changed pte high. We might have gotten rubbish values from * pte_low and pte_high, but we are guaranteed that pte_low will not have the * present bit set *unless* it is 'l'. Because get_user_pages_fast() only * operates on present ptes we're safe. */static inline pte_t ptep_get_lockless(pte_t *ptep){ pte_t pte;
do { pte.pte_low = ptep->pte_low; smp_rmb(); pte.pte_high = ptep->pte_high; smp_rmb(); } while (unlikely(pte.pte_low != ptep->pte_low));
return pte;}#define ptep_get_lockless ptep_get_lockless
#if CONFIG_PGTABLE_LEVELS > 2static inline pmd_t pmdp_get_lockless(pmd_t *pmdp){ pmd_t pmd;
do { pmd.pmd_low = pmdp->pmd_low; smp_rmb(); pmd.pmd_high = pmdp->pmd_high; smp_rmb(); } while (unlikely(pmd.pmd_low != pmdp->pmd_low));
return pmd;}#define pmdp_get_lockless pmdp_get_lockless#define pmdp_get_lockless_sync() tlb_remove_table_sync_one()#endif /* CONFIG_PGTABLE_LEVELS > 2 */#endif /* CONFIG_GUP_GET_PXX_LOW_HIGH */
/* * We require that the PTE can be read atomically. */#ifndef ptep_get_locklessstatic inline pte_t ptep_get_lockless(pte_t *ptep){ return ptep_get(ptep);}#endif
#ifndef pmdp_get_locklessstatic inline pmd_t pmdp_get_lockless(pmd_t *pmdp){ return pmdp_get(pmdp);}static inline void pmdp_get_lockless_sync(void){}#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEARstatic inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pmd_t *pmdp){ pmd_t pmd = *pmdp;
pmd_clear(pmdp); page_table_check_pmd_clear(mm, address, pmd);
return pmd;}#endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEARstatic inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pud_t *pudp){ pud_t pud = *pudp;
pud_clear(pudp); page_table_check_pud_clear(mm, address, pud);
return pud;}#endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULLstatic inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, int full){ return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);}#endif
#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULLstatic inline pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, int full){ return pudp_huge_get_and_clear(vma->vm_mm, address, pudp);}#endif#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULLstatic inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long address, pte_t *ptep, int full){ return ptep_get_and_clear(mm, address, ptep);}#endif
#ifndef get_and_clear_full_ptes/** * get_and_clear_full_ptes - Clear present PTEs that map consecutive pages of * the same folio, collecting dirty/accessed bits. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * @full: Whether we are clearing a full mm. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_get_and_clear_full(), merging dirty/accessed bits into the * returned PTE. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr, int full){ pte_t pte, tmp_pte;
pte = ptep_get_and_clear_full(mm, addr, ptep, full); while (--nr) { ptep++; addr += PAGE_SIZE; tmp_pte = ptep_get_and_clear_full(mm, addr, ptep, full); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); if (pte_young(tmp_pte)) pte = pte_mkyoung(pte); } return pte;}#endif
/** * get_and_clear_ptes - Clear present PTEs that map consecutive pages of * the same folio, collecting dirty/accessed bits. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * * Use this instead of get_and_clear_full_ptes() if it is known that we don't * need to clear the full mm, which is mostly the case. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline pte_t get_and_clear_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ return get_and_clear_full_ptes(mm, addr, ptep, nr, 0);}
#ifndef clear_full_ptes/** * clear_full_ptes - Clear present PTEs that map consecutive pages of the same * folio. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * @full: Whether we are clearing a full mm. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_get_and_clear_full(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr, int full){ for (;;) { ptep_get_and_clear_full(mm, addr, ptep, full); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
/** * clear_ptes - Clear present PTEs that map consecutive pages of the same folio. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * * Use this instead of clear_full_ptes() if it is known that we don't need to * clear the full mm, which is mostly the case. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void clear_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ clear_full_ptes(mm, addr, ptep, nr, 0);}
/* * If two threads concurrently fault at the same page, the thread that * won the race updates the PTE and its local TLB/Cache. The other thread * gives up, simply does nothing, and continues; on architectures where * software can update TLB, local TLB can be updated here to avoid next page * fault. This function updates TLB only, do nothing with cache or others. * It is the difference with function update_mmu_cache. */#ifndef update_mmu_tlb_rangestatic inline void update_mmu_tlb_range(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr){}#endif
static inline void update_mmu_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep){ update_mmu_tlb_range(vma, address, ptep, 1);}
/* * Some architectures may be able to avoid expensive synchronization * primitives when modifications are made to PTE's which are already * not present, or in the process of an address space destruction. */#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULLstatic inline void pte_clear_not_present_full(struct mm_struct *mm, unsigned long address, pte_t *ptep, int full){ pte_clear(mm, address, ptep);}#endif
#ifndef clear_not_present_full_ptes/** * clear_not_present_full_ptes - Clear multiple not present PTEs which are * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * @full: Whether we are clearing a full mm. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over pte_clear_not_present_full(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */static inline void clear_not_present_full_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr, int full){ for (;;) { pte_clear_not_present_full(mm, addr, ptep, full); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSHextern pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, pte_t *ptep);#endif
#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSHextern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, pud_t *pudp);#endif
#ifndef pte_mkwritestatic inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma){ return pte_mkwrite_novma(pte);}#endif
#if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite)static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma){ return pmd_mkwrite_novma(pmd);}#endif
#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECTstruct mm_struct;static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep){ pte_t old_pte = ptep_get(ptep); set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));}#endif
#ifndef ptep_try_set/** * ptep_try_set - atomically set an empty kernel PTE * @ptep: page table entry * @new_pte: value to install * * Atomically set *@ptep to @new_pte iff *@ptep is pte_none(). Return true on * success, false if the slot was already populated or the arch has no * implementation. * * For special kernel page tables only - never user page tables. The caller must * prevent concurrent teardown of @ptep and must accept that other writers may * race. Concurrent clearers must use ptep_get_and_clear() so racing accesses * agree on the outcome. * * Architectures opt in by providing a cmpxchg-based override and defining * ptep_try_set as an identity macro. The generic stub returns false, which is * correct for callers that fall through to oops on failure. */static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte){ return false;}#endif
#ifndef flush_tlb_before_set/** * flush_tlb_before_set - invalidate a kernel PTE's TLB before re-setting it * @addr: kernel virtual address whose PTE was just cleared * * Some architectures (e.g. arm64) do not allow a live page-table entry to be * repointed at a different page in one step. The old entry must first be made * invalid and its translation flushed from every TLB, and only then may the new * entry be written. * * This is only for the lockless atomic kernel-PTE installers (ptep_try_set()). * It must be callable with interrupts disabled. */static inline void flush_tlb_before_set(unsigned long addr){}#endif
#ifndef wrprotect_ptes/** * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same * folio. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to write-protect. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_set_wrprotect(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ for (;;) { ptep_set_wrprotect(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
#ifndef clear_flush_young_ptes/** * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same * folio as old and flush the TLB. * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear access bit. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_clear_flush_young(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline bool clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr){ bool young = false;
for (;;) { young |= ptep_clear_flush_young(vma, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }
return young;}#endif
#ifndef test_and_clear_young_ptes/** * test_and_clear_young_ptes - Mark PTEs that map consecutive pages of the same * folio as old * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear access bit. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_test_and_clear_young(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. * * Returns: whether any PTE was young. */static inline bool test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr){ bool young = false;
for (;;) { young |= ptep_test_and_clear_young(vma, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }
return young;}#endif
/* * On some architectures hardware does not set page access bit when accessing * memory page, it is responsibility of software setting this bit. It brings * out extra page fault penalty to track page access bit. For optimization page * access bit can be set during all page fault flow on these arches. * To be differentiate with macro pte_mkyoung, this macro is used on platforms * where software maintains page access bit. */#ifndef pte_sw_mkyoungstatic inline pte_t pte_sw_mkyoung(pte_t pte){ return pte;}#define pte_sw_mkyoung pte_sw_mkyoung#endif
#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT#ifdef CONFIG_TRANSPARENT_HUGEPAGEstatic inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long address, pmd_t *pmdp){ pmd_t old_pmd = *pmdp; set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));}#elsestatic inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long address, pmd_t *pmdp){ BUILD_BUG();}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD#ifdef CONFIG_TRANSPARENT_HUGEPAGEstatic inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp){ pud_t old_pud = *pudp;
set_pud_at(mm, address, pudp, pud_wrprotect(old_pud));}#elsestatic inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp){ BUILD_BUG();}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */#endif
#ifndef pmdp_collapse_flush#ifdef CONFIG_TRANSPARENT_HUGEPAGEextern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#elsestatic inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ BUILD_BUG(); return *pmdp;}#define pmdp_collapse_flush pmdp_collapse_flush#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif
#ifndef __HAVE_ARCH_PGTABLE_DEPOSITextern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable);#endif
#ifndef __HAVE_ARCH_PGTABLE_WITHDRAWextern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);#endif
#ifndef arch_needs_pgtable_deposit#define arch_needs_pgtable_deposit() (false)#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE/* * This is an implementation of pmdp_establish() that is only suitable for an * architecture that doesn't have hardware dirty/accessed bits. In this case we * can't race with CPU which sets these bits and non-atomic approach is fine. */static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd){ pmd_t old_pmd = *pmdp; set_pmd_at(vma->vm_mm, address, pmdp, pmd); return old_pmd;}#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATEextern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
/* * pmdp_invalidate_ad() invalidates the PMD while changing a transparent * hugepage mapping in the page tables. This function is similar to * pmdp_invalidate(), but should only be used if the access and dirty bits would * not be cleared by the software in the new PMD value. The function ensures * that hardware changes of the access and dirty bits updates would not be lost. * * Doing so can allow in certain architectures to avoid a TLB flush in most * cases. Yet, another TLB flush might be necessary later if the PMD update * itself requires such flush (e.g., if protection was set to be stricter). Yet, * even when a TLB flush is needed because of the update, the caller may be able * to batch these TLB flushing operations, so fewer TLB flush operations are * needed. */extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#endif
#ifndef __HAVE_ARCH_PTE_SAMEstatic inline int pte_same(pte_t pte_a, pte_t pte_b){ return pte_val(pte_a) == pte_val(pte_b);}#endif
#ifndef __HAVE_ARCH_PTE_UNUSED/* * Some architectures provide facilities to virtualization guests * so that they can flag allocated pages as unused. This allows the * host to transparently reclaim unused pages. This function returns * whether the pte's page is unused. */static inline int pte_unused(pte_t pte){ return 0;}#endif
#ifndef pte_access_permitted#define pte_access_permitted(pte, write) \ (pte_present(pte) && (!(write) || pte_write(pte)))#endif
#ifndef pmd_access_permitted#define pmd_access_permitted(pmd, write) \ (pmd_present(pmd) && (!(write) || pmd_write(pmd)))#endif
#ifndef pud_access_permitted#define pud_access_permitted(pud, write) \ (pud_present(pud) && (!(write) || pud_write(pud)))#endif
#ifndef p4d_access_permitted#define p4d_access_permitted(p4d, write) \ (p4d_present(p4d) && (!(write) || p4d_write(p4d)))#endif
#ifndef pgd_access_permitted#define pgd_access_permitted(pgd, write) \ (pgd_present(pgd) && (!(write) || pgd_write(pgd)))#endif
#ifndef __HAVE_ARCH_PMD_SAMEstatic inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b){ return pmd_val(pmd_a) == pmd_val(pmd_b);}#endif
#ifndef pud_samestatic inline int pud_same(pud_t pud_a, pud_t pud_b){ return pud_val(pud_a) == pud_val(pud_b);}#define pud_same pud_same#endif
#ifndef __HAVE_ARCH_P4D_SAMEstatic inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b){ return p4d_val(p4d_a) == p4d_val(p4d_b);}#endif
#ifndef __HAVE_ARCH_PGD_SAMEstatic inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b){ return pgd_val(pgd_a) == pgd_val(pgd_b);}#endif
#ifndef __HAVE_ARCH_DO_SWAP_PAGEstatic inline void arch_do_swap_page_nr(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pte_t pte, pte_t oldpte, int nr){
}#else/* * Some architectures support metadata associated with a page. When a * page is being swapped out, this metadata must be saved so it can be * restored when the page is swapped back in. SPARC M7 and newer * processors support an ADI (Application Data Integrity) tag for the * page as metadata for the page. arch_do_swap_page() can restore this * metadata when a page is swapped back in. */static inline void arch_do_swap_page_nr(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pte_t pte, pte_t oldpte, int nr){ for (int i = 0; i < nr; i++) { arch_do_swap_page(vma->vm_mm, vma, addr + i * PAGE_SIZE, pte_advance_pfn(pte, i), pte_advance_pfn(oldpte, i)); }}#endif
#ifndef __HAVE_ARCH_UNMAP_ONE/* * Some architectures support metadata associated with a page. When a * page is being swapped out, this metadata must be saved so it can be * restored when the page is swapped back in. SPARC M7 and newer * processors support an ADI (Application Data Integrity) tag for the * page as metadata for the page. arch_unmap_one() can save this * metadata on a swap-out of a page. */static inline int arch_unmap_one(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pte_t orig_pte){ return 0;}#endif
/* * Allow architectures to preserve additional metadata associated with * swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function * prototypes must be defined in the arch-specific asm/pgtable.h file. */#ifndef __HAVE_ARCH_PREPARE_TO_SWAPstatic inline int arch_prepare_to_swap(struct folio *folio){ return 0;}#endif
#ifndef __HAVE_ARCH_SWAP_INVALIDATEstatic inline void arch_swap_invalidate_page(int type, pgoff_t offset){}
static inline void arch_swap_invalidate_area(int type){}#endif
#ifndef __HAVE_ARCH_SWAP_RESTOREstatic inline void arch_swap_restore(swp_entry_t entry, struct folio *folio){}#endif
#ifndef __HAVE_ARCH_MOVE_PTE#define move_pte(pte, old_addr, new_addr) (pte)#endif
#ifndef pte_accessible# define pte_accessible(mm, pte) ((void)(pte), 1)#endif
#ifndef flush_tlb_fix_spurious_fault#define flush_tlb_fix_spurious_fault(vma, address, ptep) flush_tlb_page(vma, address)#endif
#ifndef flush_tlb_fix_spurious_fault_pmd#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) do { } while (0)#endif
/* * When walking page tables, get the address of the next boundary, * or the end address of the range if that comes earlier. Although no * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. */
#define pgd_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})
#ifndef p4d_addr_end#define p4d_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})#endif
#ifndef pud_addr_end#define pud_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})#endif
#ifndef pmd_addr_end#define pmd_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})#endif
/* * When walking page tables, we usually want to skip any p?d_none entries; * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */void pgd_clear_bad(pgd_t *);
#ifndef __PAGETABLE_P4D_FOLDEDvoid p4d_clear_bad(p4d_t *);#else#define p4d_clear_bad(p4d) do { } while (0)#endif
#ifndef __PAGETABLE_PUD_FOLDEDvoid pud_clear_bad(pud_t *);#else#define pud_clear_bad(p4d) do { } while (0)#endif
void pmd_clear_bad(pmd_t *);
static inline int pgd_none_or_clear_bad(pgd_t *pgd){ if (pgd_none(*pgd)) return 1; if (unlikely(pgd_bad(*pgd))) { pgd_clear_bad(pgd); return 1; } return 0;}
static inline int p4d_none_or_clear_bad(p4d_t *p4d){ if (p4d_none(*p4d)) return 1; if (unlikely(p4d_bad(*p4d))) { p4d_clear_bad(p4d); return 1; } return 0;}
static inline int pud_none_or_clear_bad(pud_t *pud){ if (pud_none(*pud)) return 1; if (unlikely(pud_bad(*pud))) { pud_clear_bad(pud); return 1; } return 0;}
static inline int pmd_none_or_clear_bad(pmd_t *pmd){ if (pmd_none(*pmd)) return 1; if (unlikely(pmd_bad(*pmd))) { pmd_clear_bad(pmd); return 1; } return 0;}
static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep){ /* * Get the current pte state, but zero it out to make it * non-present, preventing the hardware from asynchronously * updating it. */ return ptep_get_and_clear(vma->vm_mm, addr, ptep);}
static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte){ /* * The pte is non-present, so there's no hardware state to * preserve. */ set_pte_at(vma->vm_mm, addr, ptep, pte);}
#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION/* * Start a pte protection read-modify-write transaction, which * protects against asynchronous hardware modifications to the pte. * The intention is not to prevent the hardware from making pte * updates, but to prevent any updates it may make from being lost. * * This does not protect against other software modifications of the * pte; the appropriate pte lock must be held over the transaction. * * Note that this interface is intended to be batchable, meaning that * ptep_modify_prot_commit may not actually update the pte, but merely * queue the update to be done at some later time. The update must be * actually committed before the pte lock is released, however. */static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep){ return __ptep_modify_prot_start(vma, addr, ptep);}
/* * Commit an update to a pte, leaving any hardware-controlled bits in * the PTE unmodified. The pte returned from ptep_modify_prot_start() may * additionally have young and/or dirty bits set where previously they were not, * so the updated pte may have these additional changes. */static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte){ __ptep_modify_prot_commit(vma, addr, ptep, pte);}#endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
/** * modify_prot_start_ptes - Start a pte protection read-modify-write transaction * over a batch of ptes, which protects against asynchronous hardware * modifications to the ptes. The intention is not to prevent the hardware from * making pte updates, but to prevent any updates it may make from being lost. * Please see the comment above ptep_modify_prot_start() for full description. * * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_modify_prot_start(), collecting the a/d bits from each pte * in the batch. * * Note that PTE bits in the PTE batch besides the PFN can differ. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. All other PTE bits must be identical for * all PTEs in the batch except for young and dirty bits. The PTEs are all in * the same PMD. */#ifndef modify_prot_start_ptesstatic inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr){ pte_t pte, tmp_pte;
pte = ptep_modify_prot_start(vma, addr, ptep); while (--nr) { ptep++; addr += PAGE_SIZE; tmp_pte = ptep_modify_prot_start(vma, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); if (pte_young(tmp_pte)) pte = pte_mkyoung(pte); } return pte;}#endif
/** * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any * hardware-controlled bits in the PTE unmodified. * * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @old_pte: Old page table entry (for the first entry) which is now cleared. * @pte: New page table entry to be set. * @nr: Number of entries. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_modify_prot_commit(). * * Context: The caller holds the page table lock. The PTEs are all in the same * PMD. On exit, the set ptes in the batch map the same folio. The ptes set by * ptep_modify_prot_start() may additionally have young and/or dirty bits set * where previously they were not, so the updated ptes may have these * additional changes. */#ifndef modify_prot_commit_ptesstatic inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr){ int i;
for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE) { ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte);
/* Advance PFN only, set same prot */ old_pte = pte_next_pfn(old_pte); pte = pte_next_pfn(pte); }}#endif
/* * Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values * and let generic vmalloc, ioremap and page table update code know when * arch_sync_kernel_mappings() needs to be called. */#ifndef ARCH_PAGE_TABLE_SYNC_MASK#define ARCH_PAGE_TABLE_SYNC_MASK 0#endif
/* * There is no default implementation for arch_sync_kernel_mappings(). It is * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK * is 0. */void arch_sync_kernel_mappings(unsigned long start, unsigned long end);
#endif /* CONFIG_MMU */
/* * On almost all architectures and configurations, 0 can be used as the * upper ceiling to free_pgtables(): on many architectures it has the same * effect as using TASK_SIZE. However, there is one configuration which * must impose a more careful limit, to avoid freeing kernel pgtables. */#ifndef USER_PGTABLES_CEILING#define USER_PGTABLES_CEILING 0UL#endif
/* * This defines the first usable user address. Platforms * can override its value with custom FIRST_USER_ADDRESS * defined in their respective <asm/pgtable.h>. */#ifndef FIRST_USER_ADDRESS#define FIRST_USER_ADDRESS 0UL#endif
/* * No-op macros that just return the current protection value. Defined here * because these macros can be used even if CONFIG_MMU is not defined. */
#ifndef pgprot_nx#define pgprot_nx(prot) (prot)#endif
#ifndef pgprot_noncached#define pgprot_noncached(prot) (prot)#endif
#ifndef pgprot_writecombine#define pgprot_writecombine pgprot_noncached#endif
#ifndef pgprot_writethrough#define pgprot_writethrough pgprot_noncached#endif
#ifndef pgprot_device#define pgprot_device pgprot_noncached#endif
#ifndef pgprot_mhp#define pgprot_mhp(prot) (prot)#endif
#ifdef CONFIG_MMU#ifndef pgprot_modify#define pgprot_modify pgprot_modifystatic inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot){ if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) newprot = pgprot_noncached(newprot); if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) newprot = pgprot_writecombine(newprot); if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) newprot = pgprot_device(newprot); return newprot;}#endif#endif /* CONFIG_MMU */
#ifndef pgprot_encrypted#define pgprot_encrypted(prot) (prot)#endif
#ifndef pgprot_decrypted#define pgprot_decrypted(prot) (prot)#endif
/* * A facility to provide batching of the reload of page tables and * other process state with the actual context switch code for * paravirtualized guests. By convention, only one of the batched * update (lazy) modes (CPU, MMU) should be active at any given time, * entry should never be nested, and entry and exits should always be * paired. This is for sanity of maintaining and reasoning about the * kernel code. In this case, the exit (end of the context switch) is * in architecture-specific code, and so doesn't need a generic * definition. */#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH#define arch_start_context_switch(prev) do {} while (0)#endif
/* * Some platforms can customize the PTE soft-dirty bit making it unavailable * even if the architecture provides the resource. * Adding this API allows architectures to add their own checks for the * devices on which the kernel is running. * Note: When overriding it, please make sure the CONFIG_MEM_SOFT_DIRTY * is part of this macro. */#ifndef pgtable_supports_soft_dirty#define pgtable_supports_soft_dirty() IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)#endif
#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATIONstatic inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd){ return pmd;}
static inline int pmd_swp_soft_dirty(pmd_t pmd){ return 0;}
static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd){ return pmd;}#endif#else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */static inline int pte_soft_dirty(pte_t pte){ return 0;}
static inline int pmd_soft_dirty(pmd_t pmd){ return 0;}
static inline pte_t pte_mksoft_dirty(pte_t pte){ return pte;}
static inline pmd_t pmd_mksoft_dirty(pmd_t pmd){ return pmd;}
static inline pte_t pte_clear_soft_dirty(pte_t pte){ return pte;}
static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd){ return pmd;}
static inline pte_t pte_swp_mksoft_dirty(pte_t pte){ return pte;}
static inline int pte_swp_soft_dirty(pte_t pte){ return 0;}
static inline pte_t pte_swp_clear_soft_dirty(pte_t pte){ return pte;}
static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd){ return pmd;}
static inline int pmd_swp_soft_dirty(pmd_t pmd){ return 0;}
static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd){ return pmd;}#endif
#ifndef __HAVE_PFNMAP_TRACKING/* * Interfaces that can be used by architecture code to keep track of * memory type of pfn mappings specified by the remap_pfn_range, * vmf_insert_pfn. */
static inline int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, pgprot_t *prot){ return 0;}
static inline int pfnmap_track(unsigned long pfn, unsigned long size, pgprot_t *prot){ return 0;}
static inline void pfnmap_untrack(unsigned long pfn, unsigned long size){}#else/** * pfnmap_setup_cachemode - setup the cachemode in the pgprot for a pfn range * @pfn: the start of the pfn range * @size: the size of the pfn range in bytes * @prot: the pgprot to modify * * Lookup the cachemode for the pfn range starting at @pfn with the size * @size and store it in @prot, leaving other data in @prot unchanged. * * This allows for a hardware implementation to have fine-grained control of * memory cache behavior at page level granularity. Without a hardware * implementation, this function does nothing. * * Currently there is only one implementation for this - x86 Page Attribute * Table (PAT). See Documentation/arch/x86/pat.rst for more details. * * This function can fail if the pfn range spans pfns that require differing * cachemodes. If the pfn range was previously verified to have a single * cachemode, it is sufficient to query only a single pfn. The assumption is * that this is the case for drivers using the vmf_insert_pfn*() interface. * * Returns 0 on success and -EINVAL on error. */int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, pgprot_t *prot);
/** * pfnmap_track - track a pfn range * @pfn: the start of the pfn range * @size: the size of the pfn range in bytes * @prot: the pgprot to track * * Requested the pfn range to be 'tracked' by a hardware implementation and * setup the cachemode in @prot similar to pfnmap_setup_cachemode(). * * This allows for fine-grained control of memory cache behaviour at page * level granularity. Tracking memory this way is persisted across VMA splits * (VMA merging does not apply for VM_PFNMAP). * * Currently, there is only one implementation for this - x86 Page Attribute * Table (PAT). See Documentation/arch/x86/pat.rst for more details. * * Returns 0 on success and -EINVAL on error. */int pfnmap_track(unsigned long pfn, unsigned long size, pgprot_t *prot);
/** * pfnmap_untrack - untrack a pfn range * @pfn: the start of the pfn range * @size: the size of the pfn range in bytes * * Untrack a pfn range previously tracked through pfnmap_track(). */void pfnmap_untrack(unsigned long pfn, unsigned long size);#endif
/** * pfnmap_setup_cachemode_pfn - setup the cachemode in the pgprot for a pfn * @pfn: the pfn * @prot: the pgprot to modify * * Lookup the cachemode for @pfn and store it in @prot, leaving other * data in @prot unchanged. * * See pfnmap_setup_cachemode() for details. */static inline void pfnmap_setup_cachemode_pfn(unsigned long pfn, pgprot_t *prot){ pfnmap_setup_cachemode(pfn, PAGE_SIZE, prot);}
/* * ZERO_PAGE() is global shared page(s) that is always zero. It is used for * zero-mapped memory areas, CoW etc. * * On architectures that __HAVE_COLOR_ZERO_PAGE there are several such pages * for different ranges in the virtual address space. * * zero_page_pfn identifies the first (or the only) pfn for these pages. * * For architectures that don't __HAVE_COLOR_ZERO_PAGE the zero page lives in * empty_zero_page in BSS. */void arch_setup_zero_pages(void);
#ifdef __HAVE_COLOR_ZERO_PAGEstatic inline int is_zero_pfn(unsigned long pfn){ extern unsigned long zero_page_pfn; unsigned long offset_from_zero_pfn = pfn - zero_page_pfn;
return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);}
#define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
#elsestatic inline int is_zero_pfn(unsigned long pfn){ extern unsigned long zero_page_pfn;
return pfn == zero_page_pfn;}
static inline unsigned long zero_pfn(unsigned long addr){ extern unsigned long zero_page_pfn;
return zero_page_pfn;}
extern const uint8_t empty_zero_page[PAGE_SIZE];extern struct page *__zero_page;
static inline struct page *_zero_page(unsigned long addr){ return __zero_page;}#define ZERO_PAGE(vaddr) _zero_page(vaddr)
#endif /* __HAVE_COLOR_ZERO_PAGE */
#ifdef CONFIG_MMU
#ifndef CONFIG_TRANSPARENT_HUGEPAGEstatic inline int pmd_trans_huge(pmd_t pmd){ return 0;}#ifndef pmd_writestatic inline int pmd_write(pmd_t pmd){ BUG(); return 0;}#endif /* pmd_write */#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifndef pud_writestatic inline int pud_write(pud_t pud){ BUG(); return 0;}#endif /* pud_write */
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)static inline int pud_trans_huge(pud_t pud){ return 0;}#endif
static inline int pud_trans_unstable(pud_t *pud){#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) pud_t pudval = pudp_get(pud);
if (pud_none(pudval) || pud_trans_huge(pudval)) return 1; if (unlikely(pud_bad(pudval))) { pud_clear_bad(pud); return 1; }#endif return 0;}
#ifndef CONFIG_NUMA_BALANCING/* * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is * perfectly valid to indicate "no" in that case, which is why our default * implementation defaults to "always no". * * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE * page protection due to NUMA hinting. NUMA hinting faults only apply in * accessible VMAs. * * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, * looking at the VMA accessibility is sufficient. */static inline int pte_protnone(pte_t pte){ return 0;}
static inline int pmd_protnone(pmd_t pmd){ return 0;}#endif /* CONFIG_NUMA_BALANCING */
#endif /* CONFIG_MMU */
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
#ifndef __PAGETABLE_P4D_FOLDEDint p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);void p4d_clear_huge(p4d_t *p4d);#elsestatic inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot){ return 0;}static inline void p4d_clear_huge(p4d_t *p4d) { }#endif /* !__PAGETABLE_P4D_FOLDED */
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);int pud_clear_huge(pud_t *pud);int pmd_clear_huge(pmd_t *pmd);int p4d_free_pud_page(p4d_t *p4d, unsigned long addr);int pud_free_pmd_page(pud_t *pud, unsigned long addr);int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot){ return 0;}static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot){ return 0;}static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot){ return 0;}static inline void p4d_clear_huge(p4d_t *p4d) { }static inline int pud_clear_huge(pud_t *pud){ return 0;}static inline int pmd_clear_huge(pmd_t *pmd){ return 0;}static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr){ return 0;}static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr){ return 0;}static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr){ return 0;}#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE#ifdef CONFIG_TRANSPARENT_HUGEPAGE/* * ARCHes with special requirements for evicting THP backing TLB entries can * implement this. Otherwise also, it can help optimize normal TLB flush in * THP regime. Stock flush_tlb_range() typically has optimization to nuke the * entire TLB if flush span is greater than a threshold, which will * likely be true for a single huge page. Thus a single THP flush will * invalidate the entire TLB which is not desirable. * e.g. see arch/arc: flush_pmd_tlb_range */#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)#else#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()#endif#endif
struct file;int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, unsigned long size, pgprot_t *vma_prot);
#ifndef CONFIG_X86_ESPFIX64static inline void init_espfix_bsp(void) { }#endif
extern void __init pgtable_cache_init(void);
#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWEDstatic inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot){ return true;}
static inline bool arch_has_pfn_modify_check(void){ return false;}#endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */
/* * Architecture PAGE_KERNEL_* fallbacks * * Some architectures don't define certain PAGE_KERNEL_* flags. This is either * because they really don't support them, or the port needs to be updated to * reflect the required functionality. Below are a set of relatively safe * fallbacks, as best effort, which we can count on in lieu of the architectures * not defining them on their own yet. */
#ifndef PAGE_KERNEL_RO# define PAGE_KERNEL_RO PAGE_KERNEL#endif
#ifndef PAGE_KERNEL_EXEC# define PAGE_KERNEL_EXEC PAGE_KERNEL#endif
/* * Page Table Modification bits for pgtbl_mod_mask. * * These are used by the p?d_alloc_track*() and p*d_populate_kernel() * functions in the generic vmalloc, ioremap and page table update code * to track at which page-table levels entries have been modified. * Based on that the code can better decide when page table changes need * to be synchronized to other page-tables in the system. */#define __PGTBL_PGD_MODIFIED 0#define __PGTBL_P4D_MODIFIED 1#define __PGTBL_PUD_MODIFIED 2#define __PGTBL_PMD_MODIFIED 3#define __PGTBL_PTE_MODIFIED 4
#define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED)#define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED)#define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED)#define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED)#define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED)
/* Page-Table Modification Mask */typedef unsigned int pgtbl_mod_mask;
enum pgtable_level { PGTABLE_LEVEL_PTE = 0, PGTABLE_LEVEL_PMD, PGTABLE_LEVEL_PUD, PGTABLE_LEVEL_P4D, PGTABLE_LEVEL_PGD,};
static inline const char *pgtable_level_to_str(enum pgtable_level level){ switch (level) { case PGTABLE_LEVEL_PTE: return "pte"; case PGTABLE_LEVEL_PMD: return "pmd"; case PGTABLE_LEVEL_PUD: return "pud"; case PGTABLE_LEVEL_P4D: return "p4d"; case PGTABLE_LEVEL_PGD: return "pgd"; default: return "unknown"; }}
#endif /* !__ASSEMBLY__ */
#if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)#ifdef CONFIG_PHYS_ADDR_T_64BIT/* * ZSMALLOC needs to know the highest PFN on 32-bit architectures * with physical address space extension, but falls back to * BITS_PER_LONG otherwise. */#error Missing MAX_POSSIBLE_PHYSMEM_BITS definition#else#define MAX_POSSIBLE_PHYSMEM_BITS 32#endif#endif
#ifndef has_transparent_hugepage#define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)#endif
#ifndef has_transparent_pud_hugepage#define has_transparent_pud_hugepage() IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)#endif/* * On some architectures it depends on the mm if the p4d/pud or pmd * layer of the page table hierarchy is folded or not. */#ifndef mm_p4d_folded#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED)#endif
#ifndef mm_pud_folded#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED)#endif
#ifndef mm_pmd_folded#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)#endif
#ifndef p4d_offset_lockless#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)#endif#ifndef pud_offset_lockless#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)#endif#ifndef pmd_offset_lockless#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)#endif
/* * pXd_leaf() is the API to check whether a pgtable entry is a huge page * mapping. It should work globally across all archs, without any * dependency on CONFIG_* options. For architectures that do not support * huge mappings on specific levels, below fallbacks will be used. * * A leaf pgtable entry should always imply the following: * * - It is a "present" entry. IOW, before using this API, please check it * with pXd_present() first. NOTE: it may not always mean the "present * bit" is set. For example, PROT_NONE entries are always "present". * * - It should _never_ be a swap entry of any type. Above "present" check * should have guarded this, but let's be crystal clear on this. * * - It should contain a huge PFN, which points to a huge page larger than * PAGE_SIZE of the platform. The PFN format isn't important here. * * - It should cover all kinds of huge mappings (i.e. pXd_trans_huge() * or hugetlb mappings). */#ifndef pgd_leaf#define pgd_leaf(x) false#endif#ifndef p4d_leaf#define p4d_leaf(x) false#endif#ifndef pud_leaf#define pud_leaf(x) false#endif#ifndef pmd_leaf#define pmd_leaf(x) false#endif
#ifndef pgd_leaf_size#define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT)#endif#ifndef p4d_leaf_size#define p4d_leaf_size(x) P4D_SIZE#endif#ifndef pud_leaf_size#define pud_leaf_size(x) PUD_SIZE#endif#ifndef pmd_leaf_size#define pmd_leaf_size(x) PMD_SIZE#endif#ifndef __pte_leaf_size#ifndef pte_leaf_size#define pte_leaf_size(x) PAGE_SIZE#endif#define __pte_leaf_size(x,y) pte_leaf_size(y)#endif
/* * We always define pmd_pfn for all archs as it's used in lots of generic * code. Now it happens too for pud_pfn (and can happen for larger * mappings too in the future; we're not there yet). Instead of defining * it for all archs (like pmd_pfn), provide a fallback. * * Note that returning 0 here means any arch that didn't define this can * get severely wrong when it hits a real pud leaf. It's arch's * responsibility to properly define it when a huge pud is possible. */#ifndef pud_pfn#define pud_pfn(x) 0#endif
/* * Some architectures have MMUs that are configurable or selectable at boot * time. These lead to variable PTRS_PER_x. For statically allocated arrays it * helps to have a static maximum value. */
#ifndef MAX_PTRS_PER_PTE#define MAX_PTRS_PER_PTE PTRS_PER_PTE#endif
#ifndef MAX_PTRS_PER_PMD#define MAX_PTRS_PER_PMD PTRS_PER_PMD#endif
#ifndef MAX_PTRS_PER_PUD#define MAX_PTRS_PER_PUD PTRS_PER_PUD#endif
#ifndef MAX_PTRS_PER_P4D#define MAX_PTRS_PER_P4D PTRS_PER_P4D#endif
#ifndef pte_pgprot#define pte_pgprot(x) ((pgprot_t) {0})#endif
#ifndef pmd_pgprot#define pmd_pgprot(x) ((pgprot_t) {0})#endif
#ifndef pud_pgprot#define pud_pgprot(x) ((pgprot_t) {0})#endif
/* description of effects of mapping type and prot in current implementation. * this is due to the limited x86 page protection hardware. The expected * behavior is in parens: * * map_type prot * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes * w: (no) no w: (no) no w: (yes) yes w: (no) no * x: (no) no x: (no) yes x: (no) yes x: (yes) yes * * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes * w: (no) no w: (no) no w: (copy) copy w: (no) no * x: (no) no x: (no) yes x: (no) yes x: (yes) yes * * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and * MAP_PRIVATE (with Enhanced PAN supported): * r: (no) no * w: (no) no * x: (yes) yes */#define DECLARE_VM_GET_PAGE_PROT \pgprot_t vm_get_page_prot(vm_flags_t vm_flags) \{ \ return protection_map[vm_flags & \ (VM_READ | VM_WRITE | VM_EXEC | VM_SHARED)]; \} \EXPORT_SYMBOL(vm_get_page_prot);
#endif /* _LINUX_PGTABLE_H */