Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_PGTABLE_H#define _LINUX_PGTABLE_H
#include <linux/pfn.h>#include <asm/pgtable.h>
#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)#define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT)
#ifndef __ASSEMBLER__#ifdef CONFIG_MMU
#include <linux/mm_types.h>#include <linux/bug.h>#include <linux/errno.h>#include <asm-generic/pgtable_uffd.h>#include <linux/page_table_check.h>
#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED#endif
/* * This defines the generic helper for accessing PMD page * table page. Although platforms can still override this * via their respective <asm/pgtable.h>. */#ifndef pmd_pgtable#define pmd_pgtable(pmd) pmd_page(pmd)#endif
#define pmd_folio(pmd) page_folio(pmd_page(pmd))
/* * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] * * The pXx_index() functions return the index of the entry in the page * table page which would control the given virtual address * * As these functions may be used by the same code for different levels of * the page table folding, they are always available, regardless of * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 * because in such cases PTRS_PER_PxD equals 1. */
static inline unsigned long pte_index(unsigned long address){ return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);}
#ifndef pmd_indexstatic inline unsigned long pmd_index(unsigned long address){ return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);}#define pmd_index pmd_index#endif
#ifndef pud_indexstatic inline unsigned long pud_index(unsigned long address){ return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);}#define pud_index pud_index#endif
#ifndef pgd_index/* Must be a compile-time constant, so implement it as a macro */#define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))#endif
#ifndef kernel_pte_initstatic inline void kernel_pte_init(void *addr){}#define kernel_pte_init kernel_pte_init#endif
#ifndef pmd_initstatic inline void pmd_init(void *addr){}#define pmd_init pmd_init#endif
#ifndef pud_initstatic inline void pud_init(void *addr){}#define pud_init pud_init#endif
#ifndef pte_offset_kernelstatic inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address){ return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);}#define pte_offset_kernel pte_offset_kernel#endif
#ifdef CONFIG_HIGHPTE#define __pte_map(pmd, address) \ ((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address)))#define pte_unmap(pte) do { \ kunmap_local((pte)); \ rcu_read_unlock(); \} while (0)#elsestatic inline pte_t *__pte_map(pmd_t *pmd, unsigned long address){ return pte_offset_kernel(pmd, address);}static inline void pte_unmap(pte_t *pte){ rcu_read_unlock();}#endif
void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable);
/* Find an entry in the second-level page table.. */#ifndef pmd_offsetstatic inline pmd_t *pmd_offset(pud_t *pud, unsigned long address){ return pud_pgtable(*pud) + pmd_index(address);}#define pmd_offset pmd_offset#endif
#ifndef pud_offsetstatic inline pud_t *pud_offset(p4d_t *p4d, unsigned long address){ return p4d_pgtable(*p4d) + pud_index(address);}#define pud_offset pud_offset#endif
static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address){ return (pgd + pgd_index(address));};
/* * a shortcut to get a pgd_t in a given mm */#ifndef pgd_offset#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))#endif
/* * a shortcut which implies the use of the kernel's pgd, instead * of a process's */#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
/* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a * pointer to the PMD entry in user or kernel page table or translate a virtual * address to the pointer in the PTE in the kernel page tables with simple * helpers. */static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va){ return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va);}
static inline pmd_t *pmd_off_k(unsigned long va){ return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va);}
static inline pte_t *virt_to_kpte(unsigned long vaddr){ pmd_t *pmd = pmd_off_k(vaddr);
return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);}
#ifndef pmd_youngstatic inline int pmd_young(pmd_t pmd){ return 0;}#endif
#ifndef pmd_dirtystatic inline int pmd_dirty(pmd_t pmd){ return 0;}#endif
/* * A facility to provide lazy MMU batching. This allows PTE updates and * page invalidations to be delayed until a call to leave lazy MMU mode * is issued. Some architectures may benefit from doing this, and it is * beneficial for both shadow and direct mode hypervisors, which may batch * the PTE updates which happen during this window. Note that using this * interface requires that read hazards be removed from the code. A read * hazard could result in the direct mode hypervisor case, since the actual * write to the page tables may not yet have taken place, so reads though * a raw PTE pointer after it has been modified are not guaranteed to be * up to date. * * In the general case, no lock is guaranteed to be held between entry and exit * of the lazy mode. (In practice, for user PTE updates, the appropriate page * table lock(s) are held, but for kernel PTE updates, no lock is held). * The implementation must therefore assume preemption may be enabled upon * entry to the mode and cpu migration is possible; it must take steps to be * robust against this. An implementation may handle this by disabling * preemption, as a consequence generic code may not sleep while the lazy MMU * mode is active. * * The mode is disabled in interrupt context and calls to the lazy_mmu API have * no effect. * * The lazy MMU mode is enabled for a given block of code using: * * lazy_mmu_mode_enable(); * <code> * lazy_mmu_mode_disable(); * * Nesting is permitted: <code> may itself use an enable()/disable() pair. * A nested call to enable() has no functional effect; however disable() causes * any batched architectural state to be flushed regardless of nesting. After a * call to disable(), the caller can therefore rely on all previous page table * modifications to have taken effect, but the lazy MMU mode may still be * enabled. * * In certain cases, it may be desirable to temporarily pause the lazy MMU mode. * This can be done using: * * lazy_mmu_mode_pause(); * <code> * lazy_mmu_mode_resume(); * * pause() ensures that the mode is exited regardless of the nesting level; * resume() re-enters the mode at the same nesting level. Any call to the * lazy_mmu_mode_* API between those two calls has no effect. In particular, * this means that pause()/resume() pairs may nest. * * is_lazy_mmu_mode_active() can be used to check whether the lazy MMU mode is * currently enabled. */#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE/** * lazy_mmu_mode_enable() - Enable the lazy MMU mode. * * Enters a new lazy MMU mode section; if the mode was not already enabled, * enables it and calls arch_enter_lazy_mmu_mode(). * * Must be paired with a call to lazy_mmu_mode_disable(). * * Has no effect if called: * - While paused - see lazy_mmu_mode_pause() * - In interrupt context */static inline void lazy_mmu_mode_enable(void){ struct lazy_mmu_state *state = ¤t->lazy_mmu_state;
if (in_interrupt() || state->pause_count > 0) return;
VM_WARN_ON_ONCE(state->enable_count == U8_MAX);
if (state->enable_count++ == 0) arch_enter_lazy_mmu_mode();}
/** * lazy_mmu_mode_disable() - Disable the lazy MMU mode. * * Exits the current lazy MMU mode section. If it is the outermost section, * disables the mode and calls arch_leave_lazy_mmu_mode(). Otherwise (nested * section), calls arch_flush_lazy_mmu_mode(). * * Must match a call to lazy_mmu_mode_enable(). * * Has no effect if called: * - While paused - see lazy_mmu_mode_pause() * - In interrupt context */static inline void lazy_mmu_mode_disable(void){ struct lazy_mmu_state *state = ¤t->lazy_mmu_state;
if (in_interrupt() || state->pause_count > 0) return;
VM_WARN_ON_ONCE(state->enable_count == 0);
if (--state->enable_count == 0) arch_leave_lazy_mmu_mode(); else /* Exiting a nested section */ arch_flush_lazy_mmu_mode();
}
/** * __task_lazy_mmu_mode_pause() - Pause the lazy MMU mode for a task. * @tsk: The task to check. * * Pauses the lazy MMU mode of @tsk. * * This function only operates on the state saved in task_struct; to pause * current lazy_mmu_mode_pause() should be used instead. * * This function is intended for architectures that implement the lazy MMU * mode; it must not be called from generic code. */static inline void __task_lazy_mmu_mode_pause(struct task_struct *tsk){ struct lazy_mmu_state *state = &tsk->lazy_mmu_state;
VM_WARN_ON_ONCE(state->pause_count == U8_MAX);
if (state->pause_count++ == 0 && state->enable_count > 0) arch_leave_lazy_mmu_mode();}
/** * lazy_mmu_mode_pause() - Pause the lazy MMU mode. * * Pauses the lazy MMU mode; if it is currently active, disables it and calls * arch_leave_lazy_mmu_mode(). * * Must be paired with a call to lazy_mmu_mode_resume(). Calls to the * lazy_mmu_mode_* API have no effect until the matching resume() call. * * Has no effect if called: * - While paused (inside another pause()/resume() pair) * - In interrupt context */static inline void lazy_mmu_mode_pause(void){ if (in_interrupt()) return;
__task_lazy_mmu_mode_pause(current);}
/** * __task_lazy_mmu_mode_resume() - Resume the lazy MMU mode for a task. * @tsk: The task to check. * * Resumes the lazy MMU mode of @tsk. * * This function only operates on the state saved in task_struct; to resume * current lazy_mmu_mode_resume() should be used instead. * * This function is intended for architectures that implement the lazy MMU * mode; it must not be called from generic code. */static inline void __task_lazy_mmu_mode_resume(struct task_struct *tsk){ struct lazy_mmu_state *state = &tsk->lazy_mmu_state;
VM_WARN_ON_ONCE(state->pause_count == 0);
if (--state->pause_count == 0 && state->enable_count > 0) arch_enter_lazy_mmu_mode();}
/** * lazy_mmu_mode_resume() - Resume the lazy MMU mode. * * Resumes the lazy MMU mode; if it was active at the point where the matching * call to lazy_mmu_mode_pause() was made, re-enables it and calls * arch_enter_lazy_mmu_mode(). * * Must match a call to lazy_mmu_mode_pause(). * * Has no effect if called: * - While paused (inside another pause()/resume() pair) * - In interrupt context */static inline void lazy_mmu_mode_resume(void){ if (in_interrupt()) return;
__task_lazy_mmu_mode_resume(current);}#elsestatic inline void lazy_mmu_mode_enable(void) {}static inline void lazy_mmu_mode_disable(void) {}static inline void lazy_mmu_mode_pause(void) {}static inline void lazy_mmu_mode_resume(void) {}#endif
#ifndef pte_batch_hint/** * pte_batch_hint - Number of pages that can be added to batch without scanning. * @ptep: Page table pointer for the entry. * @pte: Page table entry. * * Some architectures know that a set of contiguous ptes all map the same * contiguous memory with the same permissions. In this case, it can provide a * hint to aid pte batching without the core code needing to scan every pte. * * An architecture implementation may ignore the PTE accessed state. Further, * the dirty state must apply atomically to all the PTEs described by the hint. * * May be overridden by the architecture, else pte_batch_hint is always 1. */static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte){ return 1;}#endif
#ifndef pte_advance_pfnstatic inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr){ return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT));}#endif
#define pte_next_pfn(pte) pte_advance_pfn(pte, 1)
#ifndef set_ptes/** * set_ptes - Map consecutive pages to a contiguous range of addresses. * @mm: Address space to map the pages into. * @addr: Address to map the first page at. * @ptep: Page table pointer for the first entry. * @pte: Page table entry for the first page. * @nr: Number of pages to map. * * When nr==1, initial state of pte may be present or not present, and new state * may be present or not present. When nr>1, initial state of all ptes must be * not present, and new state must be present. * * May be overridden by the architecture, or the architecture can define * set_pte() and PFN_PTE_SHIFT. * * Context: The caller holds the page table lock. The pages all belong * to the same folio. The PTEs are all in the same PMD. */static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr){ page_table_check_ptes_set(mm, addr, ptep, pte, nr);
for (;;) { set_pte(ptep, pte); if (--nr == 0) break; ptep++; pte = pte_next_pfn(pte); }}#endif#define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1)
#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGSextern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty);#endif
#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS#ifdef CONFIG_TRANSPARENT_HUGEPAGEextern int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty);extern int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t entry, int dirty);#elsestatic inline int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty){ BUILD_BUG(); return 0;}static inline int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t entry, int dirty){ BUILD_BUG(); return 0;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif
#ifndef ptep_getstatic inline pte_t ptep_get(const pte_t *ptep){ return READ_ONCE(*ptep);}#endif
#ifndef pmdp_getstatic inline pmd_t pmdp_get(const pmd_t *pmdp){ return READ_ONCE(*pmdp);}#endif
#ifndef pudp_getstatic inline pud_t pudp_get(const pud_t *pudp){ return READ_ONCE(*pudp);}#endif
#ifndef p4dp_getstatic inline p4d_t p4dp_get(const p4d_t *p4dp){ return READ_ONCE(*p4dp);}#endif
#ifndef pgdp_getstatic inline pgd_t pgdp_get(const pgd_t *pgdp){ return READ_ONCE(*pgdp);}#endif
#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNGstatic inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep){ pte_t pte = ptep_get(ptep); bool young = true;
if (!pte_young(pte)) young = false; else set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); return young;}#endif
#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ pmd_t pmd = *pmdp; bool young = true;
if (!pmd_young(pmd)) young = false; else set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); return young;}#elsestatic inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ BUILD_BUG(); return false;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSHbool ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep);#endif
#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH#ifdef CONFIG_TRANSPARENT_HUGEPAGEbool pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#else/* * Despite relevant to THP only, this API is called from generic rmap code * under PageTransHuge(), hence needs a dummy implementation for !THP */static inline bool pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ BUILD_BUG(); return false;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif
#ifndef arch_has_hw_nonleaf_pmd_young/* * Return whether the accessed bit in non-leaf PMD entries is supported on the * local CPU. */static inline bool arch_has_hw_nonleaf_pmd_young(void){ return IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG);}#endif
#ifndef arch_has_hw_pte_young/* * Return whether the accessed bit is supported on the local CPU. * * This stub assumes accessing through an old PTE triggers a page fault. * Architectures that automatically set the access bit should overwrite it. */static inline bool arch_has_hw_pte_young(void){ return IS_ENABLED(CONFIG_ARCH_HAS_HW_PTE_YOUNG);}#endif
#ifndef exec_folio_order/* * Returns preferred minimum folio order for executable file-backed memory. Must * be in range [0, PMD_ORDER). Default to order-0. */static inline unsigned int exec_folio_order(void){ return 0;}#endif
#ifndef arch_check_zapped_ptestatic inline void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte){}#endif
#ifndef arch_check_zapped_pmdstatic inline void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd){}#endif
#ifndef arch_check_zapped_pudstatic inline void arch_check_zapped_pud(struct vm_area_struct *vma, pud_t pud){}#endif
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEARstatic inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, pte_t *ptep){ pte_t pte = ptep_get(ptep); pte_clear(mm, address, ptep); page_table_check_pte_clear(mm, address, pte); return pte;}#endif
#ifndef clear_young_dirty_ptes/** * clear_young_dirty_ptes - Mark PTEs that map consecutive pages of the * same folio as old/clean. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to mark old/clean. * @flags: Flags to modify the PTE batch semantics. * * May be overridden by the architecture; otherwise, implemented by * get_and_clear/modify/set for each pte in the range. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags){ pte_t pte;
for (;;) { if (flags == CYDP_CLEAR_YOUNG) ptep_test_and_clear_young(vma, addr, ptep); else { pte = ptep_get_and_clear(vma->vm_mm, addr, ptep); if (flags & CYDP_CLEAR_YOUNG) pte = pte_mkold(pte); if (flags & CYDP_CLEAR_DIRTY) pte = pte_mkclean(pte); set_pte_at(vma->vm_mm, addr, ptep, pte); } if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep){ pte_t pte = ptep_get(ptep);
pte_clear(mm, addr, ptep); /* * No need for ptep_get_and_clear(): page table check doesn't care about * any bits that could have been set by HW concurrently. */ page_table_check_pte_clear(mm, addr, pte);}
#ifdef CONFIG_GUP_GET_PXX_LOW_HIGH/* * For walking the pagetables without holding any locks. Some architectures * (eg x86-32 PAE) cannot load the entries atomically without using expensive * instructions. We are guaranteed that a PTE will only either go from not * present to present, or present to not present -- it will not switch to a * completely different present page without a TLB flush inbetween; which we * are blocking by holding interrupts off. * * Setting ptes from not present to present goes: * * ptep->pte_high = h; * smp_wmb(); * ptep->pte_low = l; * * And present to not present goes: * * ptep->pte_low = 0; * smp_wmb(); * ptep->pte_high = 0; * * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'. * We load pte_high *after* loading pte_low, which ensures we don't see an older * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't * picked up a changed pte high. We might have gotten rubbish values from * pte_low and pte_high, but we are guaranteed that pte_low will not have the * present bit set *unless* it is 'l'. Because get_user_pages_fast() only * operates on present ptes we're safe. */static inline pte_t ptep_get_lockless(pte_t *ptep){ pte_t pte;
do { pte.pte_low = ptep->pte_low; smp_rmb(); pte.pte_high = ptep->pte_high; smp_rmb(); } while (unlikely(pte.pte_low != ptep->pte_low));
return pte;}#define ptep_get_lockless ptep_get_lockless
#if CONFIG_PGTABLE_LEVELS > 2static inline pmd_t pmdp_get_lockless(pmd_t *pmdp){ pmd_t pmd;
do { pmd.pmd_low = pmdp->pmd_low; smp_rmb(); pmd.pmd_high = pmdp->pmd_high; smp_rmb(); } while (unlikely(pmd.pmd_low != pmdp->pmd_low));
return pmd;}#define pmdp_get_lockless pmdp_get_lockless#define pmdp_get_lockless_sync() tlb_remove_table_sync_one()#endif /* CONFIG_PGTABLE_LEVELS > 2 */#endif /* CONFIG_GUP_GET_PXX_LOW_HIGH */
/* * We require that the PTE can be read atomically. */#ifndef ptep_get_locklessstatic inline pte_t ptep_get_lockless(pte_t *ptep){ return ptep_get(ptep);}#endif
#ifndef pmdp_get_locklessstatic inline pmd_t pmdp_get_lockless(pmd_t *pmdp){ return pmdp_get(pmdp);}static inline void pmdp_get_lockless_sync(void){}#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEARstatic inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pmd_t *pmdp){ pmd_t pmd = *pmdp;
pmd_clear(pmdp); page_table_check_pmd_clear(mm, address, pmd);
return pmd;}#endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEARstatic inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pud_t *pudp){ pud_t pud = *pudp;
pud_clear(pudp); page_table_check_pud_clear(mm, address, pud);
return pud;}#endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULLstatic inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, int full){ return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);}#endif
#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULLstatic inline pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, int full){ return pudp_huge_get_and_clear(vma->vm_mm, address, pudp);}#endif#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULLstatic inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long address, pte_t *ptep, int full){ return ptep_get_and_clear(mm, address, ptep);}#endif
#ifndef get_and_clear_full_ptes/** * get_and_clear_full_ptes - Clear present PTEs that map consecutive pages of * the same folio, collecting dirty/accessed bits. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * @full: Whether we are clearing a full mm. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_get_and_clear_full(), merging dirty/accessed bits into the * returned PTE. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr, int full){ pte_t pte, tmp_pte;
pte = ptep_get_and_clear_full(mm, addr, ptep, full); while (--nr) { ptep++; addr += PAGE_SIZE; tmp_pte = ptep_get_and_clear_full(mm, addr, ptep, full); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); if (pte_young(tmp_pte)) pte = pte_mkyoung(pte); } return pte;}#endif
/** * get_and_clear_ptes - Clear present PTEs that map consecutive pages of * the same folio, collecting dirty/accessed bits. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * * Use this instead of get_and_clear_full_ptes() if it is known that we don't * need to clear the full mm, which is mostly the case. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline pte_t get_and_clear_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ return get_and_clear_full_ptes(mm, addr, ptep, nr, 0);}
#ifndef clear_full_ptes/** * clear_full_ptes - Clear present PTEs that map consecutive pages of the same * folio. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * @full: Whether we are clearing a full mm. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_get_and_clear_full(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr, int full){ for (;;) { ptep_get_and_clear_full(mm, addr, ptep, full); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
/** * clear_ptes - Clear present PTEs that map consecutive pages of the same folio. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * * Use this instead of clear_full_ptes() if it is known that we don't need to * clear the full mm, which is mostly the case. * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void clear_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ clear_full_ptes(mm, addr, ptep, nr, 0);}
/* * If two threads concurrently fault at the same page, the thread that * won the race updates the PTE and its local TLB/Cache. The other thread * gives up, simply does nothing, and continues; on architectures where * software can update TLB, local TLB can be updated here to avoid next page * fault. This function updates TLB only, do nothing with cache or others. * It is the difference with function update_mmu_cache. */#ifndef update_mmu_tlb_rangestatic inline void update_mmu_tlb_range(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr){}#endif
static inline void update_mmu_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep){ update_mmu_tlb_range(vma, address, ptep, 1);}
/** * clear_nonpresent_ptes - Clear multiple non-present PTEs which are * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */static inline void clear_nonpresent_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ (void)addr;
for (;;) { pte_clear(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}
#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSHextern pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, pte_t *ptep);#endif
#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSHextern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, pud_t *pudp);#endif
#ifndef pte_mkwritestatic inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma){ return pte_mkwrite_novma(pte);}#endif
#if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite)static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma){ return pmd_mkwrite_novma(pmd);}#endif
#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECTstruct mm_struct;static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep){ pte_t old_pte = ptep_get(ptep); set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));}#endif
#ifndef ptep_try_set/** * ptep_try_set - atomically set an empty kernel PTE * @ptep: page table entry * @new_pte: value to install * * Atomically set *@ptep to @new_pte iff *@ptep is pte_none(). Return true on * success, false if the slot was already populated or the arch has no * implementation. * * For special kernel page tables only - never user page tables. The caller must * prevent concurrent teardown of @ptep and must accept that other writers may * race. Concurrent clearers must use ptep_get_and_clear() so racing accesses * agree on the outcome. * * Architectures opt in by providing a cmpxchg-based override and defining * ptep_try_set as an identity macro. The generic stub returns false, which is * correct for callers that fall through to oops on failure. */static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte){ return false;}#endif
#ifndef flush_tlb_before_set/** * flush_tlb_before_set - invalidate a kernel PTE's TLB before re-setting it * @addr: kernel virtual address whose PTE was just cleared * * Some architectures (e.g. arm64) do not allow a live page-table entry to be * repointed at a different page in one step. The old entry must first be made * invalid and its translation flushed from every TLB, and only then may the new * entry be written. * * This is only for the lockless atomic kernel-PTE installers (ptep_try_set()). * It must be callable with interrupts disabled. */static inline void flush_tlb_before_set(unsigned long addr){}#endif
#ifndef wrprotect_ptes/** * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same * folio. * @mm: Address space the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to write-protect. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_set_wrprotect(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr){ for (;;) { ptep_set_wrprotect(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }}#endif
#ifndef clear_flush_young_ptes/** * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same * folio as old and flush the TLB. * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear access bit. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_clear_flush_young(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */static inline bool clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr){ bool young = false;
for (;;) { young |= ptep_clear_flush_young(vma, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }
return young;}#endif
#ifndef test_and_clear_young_ptes/** * test_and_clear_young_ptes - Mark PTEs that map consecutive pages of the same * folio as old * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear access bit. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_test_and_clear_young(). * * Note that PTE bits in the PTE range besides the PFN can differ. For example, * some PTEs might be write-protected. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. * * Returns: whether any PTE was young. */static inline bool test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr){ bool young = false;
for (;;) { young |= ptep_test_and_clear_young(vma, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; }
return young;}#endif
/* * On some architectures hardware does not set page access bit when accessing * memory page, it is responsibility of software setting this bit. It brings * out extra page fault penalty to track page access bit. For optimization page * access bit can be set during all page fault flow on these arches. * To be differentiate with macro pte_mkyoung, this macro is used on platforms * where software maintains page access bit. */#ifndef pte_sw_mkyoungstatic inline pte_t pte_sw_mkyoung(pte_t pte){ return pte;}#define pte_sw_mkyoung pte_sw_mkyoung#endif
#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT#ifdef CONFIG_TRANSPARENT_HUGEPAGEstatic inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long address, pmd_t *pmdp){ pmd_t old_pmd = *pmdp; set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));}#elsestatic inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long address, pmd_t *pmdp){ BUILD_BUG();}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD#ifdef CONFIG_TRANSPARENT_HUGEPAGEstatic inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp){ pud_t old_pud = *pudp;
set_pud_at(mm, address, pudp, pud_wrprotect(old_pud));}#elsestatic inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp){ BUILD_BUG();}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */#endif
#ifndef pmdp_collapse_flush#ifdef CONFIG_TRANSPARENT_HUGEPAGEextern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#elsestatic inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp){ BUILD_BUG(); return *pmdp;}#define pmdp_collapse_flush pmdp_collapse_flush#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif
#ifndef __HAVE_ARCH_PGTABLE_DEPOSITextern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable);#endif
#ifndef __HAVE_ARCH_PGTABLE_WITHDRAWextern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);#endif
#ifndef arch_needs_pgtable_deposit#define arch_needs_pgtable_deposit() (false)#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE/* * This is an implementation of pmdp_establish() that is only suitable for an * architecture that doesn't have hardware dirty/accessed bits. In this case we * can't race with CPU which sets these bits and non-atomic approach is fine. */static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd){ pmd_t old_pmd = *pmdp; set_pmd_at(vma->vm_mm, address, pmdp, pmd); return old_pmd;}#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATEextern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#endif
#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
/* * pmdp_invalidate_ad() invalidates the PMD while changing a transparent * hugepage mapping in the page tables. This function is similar to * pmdp_invalidate(), but should only be used if the access and dirty bits would * not be cleared by the software in the new PMD value. The function ensures * that hardware changes of the access and dirty bits updates would not be lost. * * Doing so can allow in certain architectures to avoid a TLB flush in most * cases. Yet, another TLB flush might be necessary later if the PMD update * itself requires such flush (e.g., if protection was set to be stricter). Yet, * even when a TLB flush is needed because of the update, the caller may be able * to batch these TLB flushing operations, so fewer TLB flush operations are * needed. */extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp);#endif
#ifndef __HAVE_ARCH_PTE_SAMEstatic inline int pte_same(pte_t pte_a, pte_t pte_b){ return pte_val(pte_a) == pte_val(pte_b);}#endif
#ifndef __HAVE_ARCH_PTE_UNUSED/* * Some architectures provide facilities to virtualization guests * so that they can flag allocated pages as unused. This allows the * host to transparently reclaim unused pages. This function returns * whether the pte's page is unused. */static inline int pte_unused(pte_t pte){ return 0;}#endif
#ifndef pte_access_permitted#define pte_access_permitted(pte, write) \ (pte_present(pte) && (!(write) || pte_write(pte)))#endif
#ifndef pmd_access_permitted#define pmd_access_permitted(pmd, write) \ (pmd_present(pmd) && (!(write) || pmd_write(pmd)))#endif
#ifndef pud_access_permitted#define pud_access_permitted(pud, write) \ (pud_present(pud) && (!(write) || pud_write(pud)))#endif
#ifndef p4d_access_permitted#define p4d_access_permitted(p4d, write) \ (p4d_present(p4d) && (!(write) || p4d_write(p4d)))#endif
#ifndef pgd_access_permitted#define pgd_access_permitted(pgd, write) \ (pgd_present(pgd) && (!(write) || pgd_write(pgd)))#endif
#ifndef __HAVE_ARCH_PMD_SAMEstatic inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b){ return pmd_val(pmd_a) == pmd_val(pmd_b);}#endif
#ifndef pud_samestatic inline int pud_same(pud_t pud_a, pud_t pud_b){ return pud_val(pud_a) == pud_val(pud_b);}#define pud_same pud_same#endif
#ifndef __HAVE_ARCH_P4D_SAMEstatic inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b){ return p4d_val(p4d_a) == p4d_val(p4d_b);}#endif
#ifndef __HAVE_ARCH_PGD_SAMEstatic inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b){ return pgd_val(pgd_a) == pgd_val(pgd_b);}#endif
#ifndef __HAVE_ARCH_DO_SWAP_PAGEstatic inline void arch_do_swap_page_nr(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pte_t pte, pte_t oldpte, int nr){
}#else/* * Some architectures support metadata associated with a page. When a * page is being swapped out, this metadata must be saved so it can be * restored when the page is swapped back in. SPARC M7 and newer * processors support an ADI (Application Data Integrity) tag for the * page as metadata for the page. arch_do_swap_page() can restore this * metadata when a page is swapped back in. */static inline void arch_do_swap_page_nr(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pte_t pte, pte_t oldpte, int nr){ for (int i = 0; i < nr; i++) { arch_do_swap_page(vma->vm_mm, vma, addr + i * PAGE_SIZE, pte_advance_pfn(pte, i), pte_advance_pfn(oldpte, i)); }}#endif
#ifndef __HAVE_ARCH_UNMAP_ONE/* * Some architectures support metadata associated with a page. When a * page is being swapped out, this metadata must be saved so it can be * restored when the page is swapped back in. SPARC M7 and newer * processors support an ADI (Application Data Integrity) tag for the * page as metadata for the page. arch_unmap_one() can save this * metadata on a swap-out of a page. */static inline int arch_unmap_one(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pte_t orig_pte){ return 0;}#endif
/* * Allow architectures to preserve additional metadata associated with * swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function * prototypes must be defined in the arch-specific asm/pgtable.h file. */#ifndef __HAVE_ARCH_PREPARE_TO_SWAPstatic inline int arch_prepare_to_swap(struct folio *folio){ return 0;}#endif
#ifndef __HAVE_ARCH_SWAP_INVALIDATEstatic inline void arch_swap_invalidate_page(int type, pgoff_t offset){}
static inline void arch_swap_invalidate_area(int type){}#endif
#ifndef __HAVE_ARCH_SWAP_RESTOREstatic inline void arch_swap_restore(swp_entry_t entry, struct folio *folio){}#endif
#ifndef __HAVE_ARCH_MOVE_PTE#define move_pte(pte, old_addr, new_addr) (pte)#endif
#ifndef pte_accessible# define pte_accessible(mm, pte) ((void)(pte), 1)#endif
#ifndef flush_tlb_fix_spurious_fault#define flush_tlb_fix_spurious_fault(vma, address, ptep) flush_tlb_page(vma, address)#endif
#ifndef flush_tlb_fix_spurious_fault_pmd#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) do { } while (0)#endif
/* * When walking page tables, get the address of the next boundary, * or the end address of the range if that comes earlier. Although no * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. */
#define pgd_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})
#ifndef p4d_addr_end#define p4d_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})#endif
#ifndef pud_addr_end#define pud_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})#endif
#ifndef pmd_addr_end#define pmd_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ (__boundary - 1 < (end) - 1)? __boundary: (end); \})#endif
/* * When walking page tables, we usually want to skip any p?d_none entries; * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */void pgd_clear_bad(pgd_t *);
#ifndef __PAGETABLE_P4D_FOLDEDvoid p4d_clear_bad(p4d_t *);#else#define p4d_clear_bad(p4d) do { } while (0)#endif
#ifndef __PAGETABLE_PUD_FOLDEDvoid pud_clear_bad(pud_t *);#else#define pud_clear_bad(p4d) do { } while (0)#endif
void pmd_clear_bad(pmd_t *);
static inline int pgd_none_or_clear_bad(pgd_t *pgd){ if (pgd_none(*pgd)) return 1; if (unlikely(pgd_bad(*pgd))) { pgd_clear_bad(pgd); return 1; } return 0;}
static inline int p4d_none_or_clear_bad(p4d_t *p4d){ if (p4d_none(*p4d)) return 1; if (unlikely(p4d_bad(*p4d))) { p4d_clear_bad(p4d); return 1; } return 0;}
static inline int pud_none_or_clear_bad(pud_t *pud){ if (pud_none(*pud)) return 1; if (unlikely(pud_bad(*pud))) { pud_clear_bad(pud); return 1; } return 0;}
static inline int pmd_none_or_clear_bad(pmd_t *pmd){ if (pmd_none(*pmd)) return 1; if (unlikely(pmd_bad(*pmd))) { pmd_clear_bad(pmd); return 1; } return 0;}
static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep){ /* * Get the current pte state, but zero it out to make it * non-present, preventing the hardware from asynchronously * updating it. */ return ptep_get_and_clear(vma->vm_mm, addr, ptep);}
static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte){ /* * The pte is non-present, so there's no hardware state to * preserve. */ set_pte_at(vma->vm_mm, addr, ptep, pte);}
#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION/* * Start a pte protection read-modify-write transaction, which * protects against asynchronous hardware modifications to the pte. * The intention is not to prevent the hardware from making pte * updates, but to prevent any updates it may make from being lost. * * This does not protect against other software modifications of the * pte; the appropriate pte lock must be held over the transaction. * * Note that this interface is intended to be batchable, meaning that * ptep_modify_prot_commit may not actually update the pte, but merely * queue the update to be done at some later time. The update must be * actually committed before the pte lock is released, however. */static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep){ return __ptep_modify_prot_start(vma, addr, ptep);}
/* * Commit an update to a pte, leaving any hardware-controlled bits in * the PTE unmodified. The pte returned from ptep_modify_prot_start() may * additionally have young and/or dirty bits set where previously they were not, * so the updated pte may have these additional changes. */static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte){ __ptep_modify_prot_commit(vma, addr, ptep, pte);}#endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
/** * modify_prot_start_ptes - Start a pte protection read-modify-write transaction * over a batch of ptes, which protects against asynchronous hardware * modifications to the ptes. The intention is not to prevent the hardware from * making pte updates, but to prevent any updates it may make from being lost. * Please see the comment above ptep_modify_prot_start() for full description. * * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @nr: Number of entries. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_modify_prot_start(), collecting the a/d bits from each pte * in the batch. * * Note that PTE bits in the PTE batch besides the PFN can differ. * * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. All other PTE bits must be identical for * all PTEs in the batch except for young and dirty bits. The PTEs are all in * the same PMD. */#ifndef modify_prot_start_ptesstatic inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr){ pte_t pte, tmp_pte;
pte = ptep_modify_prot_start(vma, addr, ptep); while (--nr) { ptep++; addr += PAGE_SIZE; tmp_pte = ptep_modify_prot_start(vma, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); if (pte_young(tmp_pte)) pte = pte_mkyoung(pte); } return pte;}#endif
/** * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any * hardware-controlled bits in the PTE unmodified. * * @vma: The virtual memory area the pages are mapped into. * @addr: Address the first page is mapped at. * @ptep: Page table pointer for the first entry. * @old_pte: Old page table entry (for the first entry) which is now cleared. * @pte: New page table entry to be set. * @nr: Number of entries. * * May be overridden by the architecture; otherwise, implemented as a simple * loop over ptep_modify_prot_commit(). * * Context: The caller holds the page table lock. The PTEs are all in the same * PMD. On exit, the set ptes in the batch map the same folio. The ptes set by * ptep_modify_prot_start() may additionally have young and/or dirty bits set * where previously they were not, so the updated ptes may have these * additional changes. */#ifndef modify_prot_commit_ptesstatic inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr){ int i;
for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE) { ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte);
/* Advance PFN only, set same prot */ old_pte = pte_next_pfn(old_pte); pte = pte_next_pfn(pte); }}#endif
/* * Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values * and let generic vmalloc, ioremap and page table update code know when * arch_sync_kernel_mappings() needs to be called. */#ifndef ARCH_PAGE_TABLE_SYNC_MASK#define ARCH_PAGE_TABLE_SYNC_MASK 0#endif
/* * There is no default implementation for arch_sync_kernel_mappings(). It is * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK * is 0. */void arch_sync_kernel_mappings(unsigned long start, unsigned long end);
#endif /* CONFIG_MMU */
/* * On almost all architectures and configurations, 0 can be used as the * upper ceiling to free_pgtables(): on many architectures it has the same * effect as using TASK_SIZE. However, there is one configuration which * must impose a more careful limit, to avoid freeing kernel pgtables. */#ifndef USER_PGTABLES_CEILING#define USER_PGTABLES_CEILING 0UL#endif
/* * This defines the first usable user address. Platforms * can override its value with custom FIRST_USER_ADDRESS * defined in their respective <asm/pgtable.h>. */#ifndef FIRST_USER_ADDRESS#define FIRST_USER_ADDRESS 0UL#endif
/* * No-op macros that just return the current protection value. Defined here * because these macros can be used even if CONFIG_MMU is not defined. */
#ifndef pgprot_nx#define pgprot_nx(prot) (prot)#endif
#ifndef pgprot_noncached#define pgprot_noncached(prot) (prot)#endif
#ifndef pgprot_writecombine#define pgprot_writecombine pgprot_noncached#endif
#ifndef pgprot_writethrough#define pgprot_writethrough pgprot_noncached#endif
#ifndef pgprot_device#define pgprot_device pgprot_noncached#endif
#ifndef pgprot_mhp#define pgprot_mhp(prot) (prot)#endif
#ifdef CONFIG_MMU#ifndef pgprot_modify#define pgprot_modify pgprot_modifystatic inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot){ if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) newprot = pgprot_noncached(newprot); if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) newprot = pgprot_writecombine(newprot); if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) newprot = pgprot_device(newprot); return newprot;}#endif#endif /* CONFIG_MMU */
#ifndef pgprot_encrypted#define pgprot_encrypted(prot) (prot)#endif
#ifndef pgprot_decrypted#define pgprot_decrypted(prot) (prot)#endif
/* * A facility to provide batching of the reload of page tables and * other process state with the actual context switch code for * paravirtualized guests. By convention, only one of the batched * update (lazy) modes (CPU, MMU) should be active at any given time, * entry should never be nested, and entry and exits should always be * paired. This is for sanity of maintaining and reasoning about the * kernel code. In this case, the exit (end of the context switch) is * in architecture-specific code, and so doesn't need a generic * definition. */#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH#define arch_start_context_switch(prev) do {} while (0)#endif
/* * Some platforms can customize the PTE soft-dirty bit making it unavailable * even if the architecture provides the resource. * Adding this API allows architectures to add their own checks for the * devices on which the kernel is running. * Note: When overriding it, please make sure the CONFIG_MEM_SOFT_DIRTY * is part of this macro. */#ifndef pgtable_supports_soft_dirty#define pgtable_supports_soft_dirty() IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)#endif
#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVESstatic inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd){ return pmd;}
static inline int pmd_swp_soft_dirty(pmd_t pmd){ return 0;}
static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd){ return pmd;}#endif#else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */static inline int pte_soft_dirty(pte_t pte){ return 0;}
static inline int pmd_soft_dirty(pmd_t pmd){ return 0;}
static inline pte_t pte_mksoft_dirty(pte_t pte){ return pte;}
static inline pmd_t pmd_mksoft_dirty(pmd_t pmd){ return pmd;}
static inline pte_t pte_clear_soft_dirty(pte_t pte){ return pte;}
static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd){ return pmd;}
static inline pte_t pte_swp_mksoft_dirty(pte_t pte){ return pte;}
static inline int pte_swp_soft_dirty(pte_t pte){ return 0;}
static inline pte_t pte_swp_clear_soft_dirty(pte_t pte){ return pte;}
static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd){ return pmd;}
static inline int pmd_swp_soft_dirty(pmd_t pmd){ return 0;}
static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd){ return pmd;}#endif
#ifndef __HAVE_PFNMAP_TRACKING/* * Interfaces that can be used by architecture code to keep track of * memory type of pfn mappings specified by the remap_pfn_range, * vmf_insert_pfn. */
static inline int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, pgprot_t *prot){ return 0;}
static inline int pfnmap_track(unsigned long pfn, unsigned long size, pgprot_t *prot){ return 0;}
static inline void pfnmap_untrack(unsigned long pfn, unsigned long size){}#else/** * pfnmap_setup_cachemode - setup the cachemode in the pgprot for a pfn range * @pfn: the start of the pfn range * @size: the size of the pfn range in bytes * @prot: the pgprot to modify * * Lookup the cachemode for the pfn range starting at @pfn with the size * @size and store it in @prot, leaving other data in @prot unchanged. * * This allows for a hardware implementation to have fine-grained control of * memory cache behavior at page level granularity. Without a hardware * implementation, this function does nothing. * * Currently there is only one implementation for this - x86 Page Attribute * Table (PAT). See Documentation/arch/x86/pat.rst for more details. * * This function can fail if the pfn range spans pfns that require differing * cachemodes. If the pfn range was previously verified to have a single * cachemode, it is sufficient to query only a single pfn. The assumption is * that this is the case for drivers using the vmf_insert_pfn*() interface. * * Returns 0 on success and -EINVAL on error. */int pfnmap_setup_cachemode(unsigned long pfn, unsigned long size, pgprot_t *prot);
/** * pfnmap_track - track a pfn range * @pfn: the start of the pfn range * @size: the size of the pfn range in bytes * @prot: the pgprot to track * * Requested the pfn range to be 'tracked' by a hardware implementation and * setup the cachemode in @prot similar to pfnmap_setup_cachemode(). * * This allows for fine-grained control of memory cache behaviour at page * level granularity. Tracking memory this way is persisted across VMA splits * (VMA merging does not apply for VM_PFNMAP). * * Currently, there is only one implementation for this - x86 Page Attribute * Table (PAT). See Documentation/arch/x86/pat.rst for more details. * * Returns 0 on success and -EINVAL on error. */int pfnmap_track(unsigned long pfn, unsigned long size, pgprot_t *prot);
/** * pfnmap_untrack - untrack a pfn range * @pfn: the start of the pfn range * @size: the size of the pfn range in bytes * * Untrack a pfn range previously tracked through pfnmap_track(). */void pfnmap_untrack(unsigned long pfn, unsigned long size);#endif
/** * pfnmap_setup_cachemode_pfn - setup the cachemode in the pgprot for a pfn * @pfn: the pfn * @prot: the pgprot to modify * * Lookup the cachemode for @pfn and store it in @prot, leaving other * data in @prot unchanged. * * See pfnmap_setup_cachemode() for details. */static inline void pfnmap_setup_cachemode_pfn(unsigned long pfn, pgprot_t *prot){ pfnmap_setup_cachemode(pfn, PAGE_SIZE, prot);}
/* * ZERO_PAGE() is global shared page(s) that is always zero. It is used for * zero-mapped memory areas, CoW etc. * * On architectures that __HAVE_COLOR_ZERO_PAGE there are several such pages * for different ranges in the virtual address space. * * zero_page_pfn identifies the first (or the only) pfn for these pages. * * For architectures that don't __HAVE_COLOR_ZERO_PAGE the zero page lives in * empty_zero_page in BSS. */void arch_setup_zero_pages(void);
#ifdef __HAVE_COLOR_ZERO_PAGEstatic inline int is_zero_pfn(unsigned long pfn){ extern unsigned long zero_page_pfn; unsigned long offset_from_zero_pfn = pfn - zero_page_pfn;
return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);}
#define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
#elsestatic inline int is_zero_pfn(unsigned long pfn){ extern unsigned long zero_page_pfn;
return pfn == zero_page_pfn;}
static inline unsigned long zero_pfn(unsigned long addr){ extern unsigned long zero_page_pfn;
return zero_page_pfn;}
extern const uint8_t empty_zero_page[PAGE_SIZE];extern struct page *__zero_page;
static inline struct page *_zero_page(unsigned long addr){ return __zero_page;}#define ZERO_PAGE(vaddr) _zero_page(vaddr)
#endif /* __HAVE_COLOR_ZERO_PAGE */
#ifdef CONFIG_MMU
#ifndef CONFIG_TRANSPARENT_HUGEPAGEstatic inline int pmd_trans_huge(pmd_t pmd){ return 0;}#ifndef pmd_writestatic inline int pmd_write(pmd_t pmd){ BUG(); return 0;}#endif /* pmd_write */#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#ifndef pud_writestatic inline int pud_write(pud_t pud){ BUG(); return 0;}#endif /* pud_write */
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)static inline int pud_trans_huge(pud_t pud){ return 0;}#endif
static inline int pud_trans_unstable(pud_t *pud){#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) pud_t pudval = pudp_get(pud);
if (pud_none(pudval) || pud_trans_huge(pudval)) return 1; if (unlikely(pud_bad(pudval))) { pud_clear_bad(pud); return 1; }#endif return 0;}
#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE/* * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It * is perfectly valid to indicate "no" in that case, which is why our * default implementation defaults to "always no". * * In an accessible VMA, pte_protnone() reliably indicates a present * PROT_NONE page protection. Today the kernel uses such PTEs for two * purposes: NUMA hinting faults, and userfaultfd RWP tracking on * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and * the VMA flag; see include/linux/userfaultfd_k.h. * * So, to reliably identify PROT_NONE PTEs that require kernel handling, * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is * sufficient. * * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero * stubs below; PAGE_NONE references that survive to runtime fire the * BUILD_BUG() fallback, since callers should have folded such paths to * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). */static inline int pte_protnone(pte_t pte){ return 0;}
static inline int pmd_protnone(pmd_t pmd){ return 0;}
#ifndef PAGE_NONE#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; })#endif#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
#endif /* CONFIG_MMU */
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
#ifndef __PAGETABLE_P4D_FOLDEDint p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);void p4d_clear_huge(p4d_t *p4d);#elsestatic inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot){ return 0;}static inline void p4d_clear_huge(p4d_t *p4d) { }#endif /* !__PAGETABLE_P4D_FOLDED */
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);int pud_clear_huge(pud_t *pud);int pmd_clear_huge(pmd_t *pmd);int p4d_free_pud_page(p4d_t *p4d, unsigned long addr);int pud_free_pmd_page(pud_t *pud, unsigned long addr);int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot){ return 0;}static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot){ return 0;}static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot){ return 0;}static inline void p4d_clear_huge(p4d_t *p4d) { }static inline int pud_clear_huge(pud_t *pud){ return 0;}static inline int pmd_clear_huge(pmd_t *pmd){ return 0;}static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr){ return 0;}static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr){ return 0;}static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr){ return 0;}#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE#ifdef CONFIG_TRANSPARENT_HUGEPAGE/* * ARCHes with special requirements for evicting THP backing TLB entries can * implement this. Otherwise also, it can help optimize normal TLB flush in * THP regime. Stock flush_tlb_range() typically has optimization to nuke the * entire TLB if flush span is greater than a threshold, which will * likely be true for a single huge page. Thus a single THP flush will * invalidate the entire TLB which is not desirable. * e.g. see arch/arc: flush_pmd_tlb_range */#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)#else#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()#endif#endif
struct file;int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, unsigned long size, pgprot_t *vma_prot);
#ifndef CONFIG_X86_ESPFIX64static inline void init_espfix_bsp(void) { }#endif
extern void __init pgtable_cache_init(void);
#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWEDstatic inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot){ return true;}
static inline bool arch_has_pfn_modify_check(void){ return false;}#endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */
/* * Architecture PAGE_KERNEL_* fallbacks * * Some architectures don't define certain PAGE_KERNEL_* flags. This is either * because they really don't support them, or the port needs to be updated to * reflect the required functionality. Below are a set of relatively safe * fallbacks, as best effort, which we can count on in lieu of the architectures * not defining them on their own yet. */
#ifndef PAGE_KERNEL_RO# define PAGE_KERNEL_RO PAGE_KERNEL#endif
#ifndef PAGE_KERNEL_EXEC# define PAGE_KERNEL_EXEC PAGE_KERNEL#endif
/* * Page Table Modification bits for pgtbl_mod_mask. * * These are used by the p?d_alloc_track*() and p*d_populate_kernel() * functions in the generic vmalloc, ioremap and page table update code * to track at which page-table levels entries have been modified. * Based on that the code can better decide when page table changes need * to be synchronized to other page-tables in the system. */#define __PGTBL_PGD_MODIFIED 0#define __PGTBL_P4D_MODIFIED 1#define __PGTBL_PUD_MODIFIED 2#define __PGTBL_PMD_MODIFIED 3#define __PGTBL_PTE_MODIFIED 4
#define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED)#define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED)#define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED)#define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED)#define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED)
/* Page-Table Modification Mask */typedef unsigned int pgtbl_mod_mask;
enum pgtable_level { PGTABLE_LEVEL_PTE = 0, PGTABLE_LEVEL_PMD, PGTABLE_LEVEL_PUD, PGTABLE_LEVEL_P4D, PGTABLE_LEVEL_PGD,};
static inline const char *pgtable_level_to_str(enum pgtable_level level){ switch (level) { case PGTABLE_LEVEL_PTE: return "pte"; case PGTABLE_LEVEL_PMD: return "pmd"; case PGTABLE_LEVEL_PUD: return "pud"; case PGTABLE_LEVEL_P4D: return "p4d"; case PGTABLE_LEVEL_PGD: return "pgd"; default: return "unknown"; }}
void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size);
#define ptval_to_str(buf, val) \ do { \ auto __val = (val); \ \ ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ } while (0)
#if defined(__SIZEOF_INT128__)#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */#else#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */#endif#endif /* !__ASSEMBLER__ */
#if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)#ifdef CONFIG_PHYS_ADDR_T_64BIT/* * ZSMALLOC needs to know the highest PFN on 32-bit architectures * with physical address space extension, but falls back to * BITS_PER_LONG otherwise. */#error Missing MAX_POSSIBLE_PHYSMEM_BITS definition#else#define MAX_POSSIBLE_PHYSMEM_BITS 32#endif#endif
#ifndef has_transparent_hugepage#define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)#endif
#ifndef has_transparent_pud_hugepage#define has_transparent_pud_hugepage() IS_BUILTIN(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)#endif/* * On some architectures it depends on the mm if the p4d/pud or pmd * layer of the page table hierarchy is folded or not. */#ifndef mm_p4d_folded#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED)#endif
#ifndef mm_pud_folded#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED)#endif
#ifndef mm_pmd_folded#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)#endif
#ifndef p4d_offset_lockless#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)#endif#ifndef pud_offset_lockless#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)#endif#ifndef pmd_offset_lockless#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)#endif
/* * pXd_leaf() is the API to check whether a pgtable entry is a huge page * mapping. It should work globally across all archs, without any * dependency on CONFIG_* options. For architectures that do not support * huge mappings on specific levels, below fallbacks will be used. * * A leaf pgtable entry should always imply the following: * * - It is a "present" entry. IOW, before using this API, please check it * with pXd_present() first. NOTE: it may not always mean the "present * bit" is set. For example, PROT_NONE entries are always "present". * * - It should _never_ be a swap entry of any type. Above "present" check * should have guarded this, but let's be crystal clear on this. * * - It should contain a huge PFN, which points to a huge page larger than * PAGE_SIZE of the platform. The PFN format isn't important here. * * - It should cover all kinds of huge mappings (i.e. pXd_trans_huge() * or hugetlb mappings). */#ifndef pgd_leaf#define pgd_leaf(x) false#endif#ifndef p4d_leaf#define p4d_leaf(x) false#endif#ifndef pud_leaf#define pud_leaf(x) false#endif#ifndef pmd_leaf#define pmd_leaf(x) false#endif
#ifndef pgd_leaf_size#define pgd_leaf_size(x) (1ULL << PGDIR_SHIFT)#endif#ifndef p4d_leaf_size#define p4d_leaf_size(x) P4D_SIZE#endif#ifndef pud_leaf_size#define pud_leaf_size(x) PUD_SIZE#endif#ifndef pmd_leaf_size#define pmd_leaf_size(x) PMD_SIZE#endif#ifndef __pte_leaf_size#ifndef pte_leaf_size#define pte_leaf_size(x) PAGE_SIZE#endif#define __pte_leaf_size(x,y) pte_leaf_size(y)#endif
/* * We always define pmd_pfn for all archs as it's used in lots of generic * code. Now it happens too for pud_pfn (and can happen for larger * mappings too in the future; we're not there yet). Instead of defining * it for all archs (like pmd_pfn), provide a fallback. * * Note that returning 0 here means any arch that didn't define this can * get severely wrong when it hits a real pud leaf. It's arch's * responsibility to properly define it when a huge pud is possible. */#ifndef pud_pfn#define pud_pfn(x) 0#endif
/* * Some architectures have MMUs that are configurable or selectable at boot * time. These lead to variable PTRS_PER_x. For statically allocated arrays it * helps to have a static maximum value. */
#ifndef MAX_PTRS_PER_PTE#define MAX_PTRS_PER_PTE PTRS_PER_PTE#endif
#ifndef MAX_PTRS_PER_PMD#define MAX_PTRS_PER_PMD PTRS_PER_PMD#endif
#ifndef MAX_PTRS_PER_PUD#define MAX_PTRS_PER_PUD PTRS_PER_PUD#endif
#ifndef MAX_PTRS_PER_P4D#define MAX_PTRS_PER_P4D PTRS_PER_P4D#endif
#ifndef pte_pgprot#define pte_pgprot(x) ((pgprot_t) {0})#endif
#ifndef pmd_pgprot#define pmd_pgprot(x) ((pgprot_t) {0})#endif
#ifndef pud_pgprot#define pud_pgprot(x) ((pgprot_t) {0})#endif
/* description of effects of mapping type and prot in current implementation. * this is due to the limited x86 page protection hardware. The expected * behavior is in parens: * * map_type prot * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes * w: (no) no w: (no) no w: (yes) yes w: (no) no * x: (no) no x: (no) yes x: (no) yes x: (yes) yes * * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes * w: (no) no w: (no) no w: (copy) copy w: (no) no * x: (no) no x: (no) yes x: (no) yes x: (yes) yes * * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and * MAP_PRIVATE (with Enhanced PAN supported): * r: (no) no * w: (no) no * x: (yes) yes */#define DECLARE_VM_GET_PAGE_PROT \pgprot_t vm_get_page_prot(vm_flags_t vm_flags) \{ \ return protection_map[vm_flags & \ (VM_READ | VM_WRITE | VM_EXEC | VM_SHARED)]; \} \EXPORT_SYMBOL(vm_get_page_prot);
#endif /* _LINUX_PGTABLE_H */