Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_MMZONE_H#define _LINUX_MMZONE_H
#ifndef __ASSEMBLER__#ifndef __GENERATING_BOUNDS_H
#include <linux/spinlock.h>#include <linux/list.h>#include <linux/list_nulls.h>#include <linux/wait.h>#include <linux/bitops.h>#include <linux/cache.h>#include <linux/threads.h>#include <linux/numa.h>#include <linux/init.h>#include <linux/seqlock.h>#include <linux/nodemask.h>#include <linux/pageblock-flags.h>#include <linux/page-flags-layout.h>#include <linux/atomic.h>#include <linux/mm_types.h>#include <linux/page-flags.h>#include <linux/local_lock.h>#include <linux/zswap.h>#include <linux/sizes.h>#include <asm/page.h>
/* Free memory management - zoned buddy allocator. */#ifndef CONFIG_ARCH_FORCE_MAX_ORDER#define MAX_PAGE_ORDER 10#else#define MAX_PAGE_ORDER CONFIG_ARCH_FORCE_MAX_ORDER#endif#define MAX_ORDER_NR_PAGES (1 << MAX_PAGE_ORDER)
#define IS_MAX_ORDER_ALIGNED(pfn) IS_ALIGNED(pfn, MAX_ORDER_NR_PAGES)
#define NR_PAGE_ORDERS (MAX_PAGE_ORDER + 1)
/* Defines the order for the number of pages that have a migrate type. */#ifndef CONFIG_PAGE_BLOCK_MAX_ORDER#define PAGE_BLOCK_MAX_ORDER MAX_PAGE_ORDER#else#define PAGE_BLOCK_MAX_ORDER CONFIG_PAGE_BLOCK_MAX_ORDER#endif /* CONFIG_PAGE_BLOCK_MAX_ORDER */
/* * The MAX_PAGE_ORDER, which defines the max order of pages to be allocated * by the buddy allocator, has to be larger or equal to the PAGE_BLOCK_MAX_ORDER, * which defines the order for the number of pages that can have a migrate type */#if (PAGE_BLOCK_MAX_ORDER > MAX_PAGE_ORDER)#error MAX_PAGE_ORDER must be >= PAGE_BLOCK_MAX_ORDER#endif
/* * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed * costly to service. That is between allocation orders which should * coalesce naturally under reasonable reclaim pressure and those which * will not. */#define PAGE_ALLOC_COSTLY_ORDER 3
#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS)/* * We don't expect any folios that exceed buddy sizes (and consequently * memory sections). */#define MAX_FOLIO_ORDER MAX_PAGE_ORDER#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)/* * Only pages within a single memory section are guaranteed to be * contiguous. By limiting folios to a single memory section, all folio * pages are guaranteed to be contiguous. */#define MAX_FOLIO_ORDER PFN_SECTION_SHIFT#elif defined(CONFIG_HUGETLB_PAGE)/* * There is no real limit on the folio size. We limit them to the maximum we * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit. */#ifdef CONFIG_64BIT#define MAX_FOLIO_ORDER (ilog2(SZ_16G) - PAGE_SHIFT)#else#define MAX_FOLIO_ORDER (ilog2(SZ_1G) - PAGE_SHIFT)#endif#else/* * Without hugetlb, gigantic folios that are bigger than a single PUD are * currently impossible. */#define MAX_FOLIO_ORDER (PUD_SHIFT - PAGE_SHIFT)#endif
#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER)
enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RECLAIMABLE, MIGRATE_PCPTYPES, /* the number of types on the pcp lists */ MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,#ifdef CONFIG_CMA /* * MIGRATE_CMA migration type is designed to mimic the way * ZONE_MOVABLE works. Only movable pages can be allocated * from MIGRATE_CMA pageblocks and page allocator never * implicitly change migration type of MIGRATE_CMA pageblock. * * The way to use it is to change migratetype of a range of * pageblocks to MIGRATE_CMA which can be done by * __free_pageblock_cma() function. */ MIGRATE_CMA, __MIGRATE_TYPE_END = MIGRATE_CMA,#else __MIGRATE_TYPE_END = MIGRATE_HIGHATOMIC,#endif#ifdef CONFIG_MEMORY_ISOLATION MIGRATE_ISOLATE, /* can't allocate from here */#endif MIGRATE_TYPES};
/* In mm/page_alloc.c; keep in sync also with show_migration_types() there */extern const char * const migratetype_names[MIGRATE_TYPES];
#ifdef CONFIG_CMA# define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)# define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)/* * __dump_folio() in mm/debug.c passes a folio pointer to on-stack struct folio, * so folio_pfn() cannot be used and pfn is needed. */# define is_migrate_cma_folio(folio, pfn) \ (get_pfnblock_migratetype(&folio->page, pfn) == MIGRATE_CMA)#else# define is_migrate_cma(migratetype) false# define is_migrate_cma_page(_page) false# define is_migrate_cma_folio(folio, pfn) false#endif
static inline bool is_migrate_movable(int mt){ return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;}
/* * Check whether a migratetype can be merged with another migratetype. * * It is only mergeable when it can fall back to other migratetypes for * allocation. See fallbacks[MIGRATE_TYPES][3] in page_alloc.c. */static inline bool migratetype_is_mergeable(int mt){ return mt < MIGRATE_PCPTYPES;}
#define for_each_free_list(list, zone, order) \ for (order = 0; order < NR_PAGE_ORDERS; order++) \ for (unsigned int __type = 0; \ __type < MIGRATE_TYPES && \ (list = &(zone)->free_area[order].free_list[__type], 1); \ __type++)
extern int page_group_by_mobility_disabled;
#define get_pageblock_migratetype(page) \ get_pfnblock_migratetype(page, page_to_pfn(page))
#define folio_migratetype(folio) \ get_pageblock_migratetype(&folio->page)
struct free_area { struct list_head free_list[MIGRATE_TYPES]; unsigned long nr_free;};
struct pglist_data;
#ifdef CONFIG_NUMAenum numa_stat_item { NUMA_HIT, /* allocated in intended node */ NUMA_MISS, /* allocated in non intended node */ NUMA_FOREIGN, /* was intended here, hit elsewhere */ NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */ NUMA_LOCAL, /* allocation from local node */ NUMA_OTHER, /* allocation from other node */ NR_VM_NUMA_EVENT_ITEMS};#else#define NR_VM_NUMA_EVENT_ITEMS 0#endif
enum zone_stat_item { NR_FREE_PAGES, NR_FREE_PAGES_BLOCKS, NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */ NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE, NR_ZONE_ACTIVE_ANON, NR_ZONE_INACTIVE_FILE, NR_ZONE_ACTIVE_FILE, NR_ZONE_UNEVICTABLE, NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages */ NR_MLOCK, /* mlock()ed pages found and moved off LRU */#if IS_ENABLED(CONFIG_ZSMALLOC) NR_ZSPAGES, /* allocated in zsmalloc */#endif NR_FREE_CMA_PAGES,#ifdef CONFIG_UNACCEPTED_MEMORY NR_UNACCEPTED,#endif NR_VM_ZONE_STAT_ITEMS };
enum node_stat_item { NR_LRU_BASE, NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */ NR_ACTIVE_ANON, /* " " " " " */ NR_INACTIVE_FILE, /* " " " " " */ NR_ACTIVE_FILE, /* " " " " " */ NR_UNEVICTABLE, /* " " " " " */ NR_SLAB_RECLAIMABLE_B, NR_SLAB_UNRECLAIMABLE_B, NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */ NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */ WORKINGSET_NODES, WORKINGSET_REFAULT_BASE, WORKINGSET_REFAULT_ANON = WORKINGSET_REFAULT_BASE, WORKINGSET_REFAULT_FILE, WORKINGSET_ACTIVATE_BASE, WORKINGSET_ACTIVATE_ANON = WORKINGSET_ACTIVATE_BASE, WORKINGSET_ACTIVATE_FILE, WORKINGSET_RESTORE_BASE, WORKINGSET_RESTORE_ANON = WORKINGSET_RESTORE_BASE, WORKINGSET_RESTORE_FILE, WORKINGSET_NODERECLAIM, NR_ANON_MAPPED, /* Mapped anonymous pages */ NR_FILE_MAPPED, /* pagecache pages mapped into pagetables. only modified from process context */ NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */ NR_SHMEM_THPS, NR_SHMEM_PMDMAPPED, NR_FILE_THPS, NR_FILE_PMDMAPPED, NR_ANON_THPS, NR_VMSCAN_WRITE, NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */ NR_DIRTIED, /* page dirtyings since bootup */ NR_WRITTEN, /* page writings since bootup */ NR_THROTTLED_WRITTEN, /* NR_WRITTEN while reclaim throttled */ NR_KERNEL_MISC_RECLAIMABLE, /* reclaimable non-slab kernel pages */ NR_FOLL_PIN_ACQUIRED, /* via: pin_user_page(), gup flag: FOLL_PIN */ NR_FOLL_PIN_RELEASED, /* pages returned via unpin_user_page() */ NR_VMALLOC, NR_KERNEL_STACK_KB, /* measured in KiB */#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) NR_KERNEL_SCS_KB, /* measured in KiB */#endif NR_PAGETABLE, /* used for pagetables */ NR_SECONDARY_PAGETABLE, /* secondary pagetables, KVM & IOMMU */#ifdef CONFIG_IOMMU_SUPPORT NR_IOMMU_PAGES, /* # of pages allocated by IOMMU */#endif#ifdef CONFIG_SWAP NR_SWAPCACHE,#endif#ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS, /* promote successfully */ /** * Candidate pages for promotion based on hint fault latency. This * counter is used to control the promotion rate and adjust the hot * threshold. */ PGPROMOTE_CANDIDATE, /** * Not rate-limited (NRL) candidate pages for those can be promoted * without considering hot threshold because of enough free pages in * fast-tier node. These promotions bypass the regular hotness checks * and do NOT influence the promotion rate-limiter or * threshold-adjustment logic. * This is for statistics/monitoring purposes. */ PGPROMOTE_CANDIDATE_NRL,#endif /* PGDEMOTE_*: pages demoted */ PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, PGDEMOTE_PROACTIVE, PGSTEAL_KSWAPD, PGSTEAL_DIRECT, PGSTEAL_KHUGEPAGED, PGSTEAL_PROACTIVE, PGSTEAL_ANON, PGSTEAL_FILE, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_KHUGEPAGED, PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, PGROTATE_ANON, PGROTATE_FILE, PGREFILL,#ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB,#endif NR_BALLOON_PAGES, NR_KERNEL_FILE_PAGES, NR_GPU_ACTIVE, /* Pages assigned to GPU objects */ NR_GPU_RECLAIM, /* Pages in shrinkable GPU pools */ NR_VM_NODE_STAT_ITEMS};
/* * Returns true if the item should be printed in THPs (/proc/vmstat * currently prints number of anon, file and shmem THPs. But the item * is charged in pages). */static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item){ if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return false;
return item == NR_ANON_THPS || item == NR_FILE_THPS || item == NR_SHMEM_THPS || item == NR_SHMEM_PMDMAPPED || item == NR_FILE_PMDMAPPED;}
/* * Returns true if the value is measured in bytes (most vmstat values are * measured in pages). This defines the API part, the internal representation * might be different. */static __always_inline bool vmstat_item_in_bytes(int idx){ /* * Global and per-node slab counters track slab pages. * It's expected that changes are multiples of PAGE_SIZE. * Internally values are stored in pages. * * Per-memcg and per-lruvec counters track memory, consumed * by individual slab objects. These counters are actually * byte-precise. */ return (idx == NR_SLAB_RECLAIMABLE_B || idx == NR_SLAB_UNRECLAIMABLE_B);}
/* * We do arithmetic on the LRU lists in various places in the code, * so it is important to keep the active lists LRU_ACTIVE higher in * the array than the corresponding inactive lists, and to keep * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists. * * This has to be kept in sync with the statistics in zone_stat_item * above and the descriptions in vmstat_text in mm/vmstat.c */#define LRU_BASE 0#define LRU_ACTIVE 1#define LRU_FILE 2
enum lru_list { LRU_INACTIVE_ANON = LRU_BASE, LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE, LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE, LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE, LRU_UNEVICTABLE, NR_LRU_LISTS};
enum vmscan_throttle_state { VMSCAN_THROTTLE_WRITEBACK, VMSCAN_THROTTLE_ISOLATED, VMSCAN_THROTTLE_NOPROGRESS, VMSCAN_THROTTLE_CONGESTED, NR_VMSCAN_THROTTLE,};
#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
static inline bool is_file_lru(enum lru_list lru){ return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);}
static inline bool is_active_lru(enum lru_list lru){ return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);}
#define WORKINGSET_ANON 0#define WORKINGSET_FILE 1#define ANON_AND_FILE 2
enum lruvec_flags { /* * An lruvec has many dirty pages backed by a congested BDI: * 1. LRUVEC_CGROUP_CONGESTED is set by cgroup-level reclaim. * It can be cleared by cgroup reclaim or kswapd. * 2. LRUVEC_NODE_CONGESTED is set by kswapd node-level reclaim. * It can only be cleared by kswapd. * * Essentially, kswapd can unthrottle an lruvec throttled by cgroup * reclaim, but not vice versa. This only applies to the root cgroup. * The goal is to prevent cgroup reclaim on the root cgroup (e.g. * memory.reclaim) to unthrottle an unbalanced node (that was throttled * by kswapd). */ LRUVEC_CGROUP_CONGESTED, LRUVEC_NODE_CONGESTED,};
#endif /* !__GENERATING_BOUNDS_H */
/* * Evictable folios are divided into multiple generations. The youngest and the * oldest generation numbers, max_seq and min_seq, are monotonically increasing. * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the * corresponding generation. The gen counter in folio->flags stores gen+1 while * a folio is on one of lrugen->folios[]. Otherwise it stores 0. * * After a folio is faulted in, the aging needs to check the accessed bit at * least twice before handing this folio over to the eviction. The first check * clears the accessed bit from the initial fault; the second check makes sure * this folio hasn't been used since then. This process, AKA second chance, * requires a minimum of two generations, hence MIN_NR_GENS. And to maintain ABI * compatibility with the active/inactive LRU, e.g., /proc/vmstat, these two * generations are considered active; the rest of generations, if they exist, * are considered inactive. See lru_gen_is_active(). * * PG_active is always cleared while a folio is on one of lrugen->folios[] so * that the sliding window needs not to worry about it. And it's set again when * a folio considered active is isolated for non-reclaiming purposes, e.g., * migration. See lru_gen_add_folio() and lru_gen_del_folio(). * * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the * number of categories of the active/inactive LRU when keeping track of * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits * in folio->flags, masked by LRU_GEN_MASK. */#define MIN_NR_GENS 2U#define MAX_NR_GENS 4U
/* * Each generation is divided into multiple tiers. A folio accessed N times * through file descriptors is in tier order_base_2(N). A folio in the first * tier (N=0,1) is marked by PG_referenced unless it was faulted in through page * tables or read ahead. A folio in the last tier (MAX_NR_TIERS-1) is marked by * PG_workingset. A folio in any other tier (1<N<5) between the first and last * is marked by additional bits of LRU_REFS_WIDTH in folio->flags. * * In contrast to moving across generations which requires the LRU lock, moving * across tiers only involves atomic operations on folio->flags and therefore * has a negligible cost in the buffered access path. In the eviction path, * comparisons of refaulted/(evicted+protected) from the first tier and the rest * infer whether folios accessed multiple times through file descriptors are * statistically hot and thus worth protecting. * * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the * number of categories of the active/inactive LRU when keeping track of * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in * folio->flags, masked by LRU_REFS_MASK. */#define MAX_NR_TIERS 4U#define LRU_TIER_MIN 0U#define LRU_TIER_MAX (MAX_NR_TIERS - 1)
#ifndef __GENERATING_BOUNDS_H
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH)
/* * For folios accessed multiple times through file descriptors, * lru_gen_inc_refs() sets additional bits of LRU_REFS_WIDTH in folio->flags * after PG_referenced, then PG_workingset after LRU_REFS_WIDTH. After all its * bits are set, i.e., LRU_REFS_FLAGS|BIT(PG_workingset), a folio is lazily * promoted into the second oldest generation in the eviction path. And when * folio_inc_gen() does that, it clears LRU_REFS_FLAGS so that * lru_gen_inc_refs() can start over. Note that for this case, LRU_REFS_MASK is * only valid when PG_referenced is set. * * For folios accessed multiple times through page tables, folio_update_gen() * from a page table walk or lru_gen_set_refs() from a rmap walk sets * PG_referenced after the accessed bit is cleared for the first time. * Thereafter, those two paths set PG_workingset and promote folios to the * youngest generation. Like folio_inc_gen(), folio_update_gen() also clears * PG_referenced. Note that for this case, LRU_REFS_MASK is not used. * * For both cases above, after PG_workingset is set on a folio, it remains until * this folio is either reclaimed, or "deactivated" by lru_gen_clear_refs(). It * can be set again if lru_gen_test_recent() returns true upon a refault. */#define LRU_REFS_FLAGS (LRU_REFS_MASK | BIT(PG_referenced))
struct lruvec;struct page_vma_mapped_walk;
#ifdef CONFIG_LRU_GEN
enum { LRU_GEN_ANON, LRU_GEN_FILE,};
enum { LRU_GEN_CORE, LRU_GEN_MM_WALK, LRU_GEN_NONLEAF_YOUNG, NR_LRU_GEN_CAPS};
#define MIN_LRU_BATCH BITS_PER_LONG#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64)
/* whether to keep historical stats from evicted generations */#ifdef CONFIG_LRU_GEN_STATS#define NR_HIST_GENS MAX_NR_GENS#else#define NR_HIST_GENS 1U#endif
/* * The youngest generation number is stored in max_seq for both anon and file * types as they are aged on an equal footing. The oldest generation numbers are * stored in min_seq[] separately for anon and file types so that they can be * incremented independently. Ideally min_seq[] are kept in sync when both anon * and file types are evictable. However, to adapt to situations like extreme * swappiness, they are allowed to be out of sync by at most * MAX_NR_GENS-MIN_NR_GENS-1. * * The number of pages in each generation is eventually consistent and therefore * can be transiently negative when reset_batch_size() is pending. */struct lru_gen_folio { /* the aging increments the youngest generation number */ unsigned long max_seq; /* the eviction increments the oldest generation numbers */ unsigned long min_seq[ANON_AND_FILE]; /* the birth time of each generation in jiffies */ unsigned long timestamps[MAX_NR_GENS]; /* the multi-gen LRU lists, lazily sorted on eviction */ struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the multi-gen LRU sizes, eventually consistent */ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the exponential moving average of refaulted */ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; /* the exponential moving average of evicted+protected */ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; /* can only be modified under the LRU lock */ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; /* can be modified without holding the LRU lock */ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; /* whether the multi-gen LRU is enabled */ bool enabled; /* the memcg generation this lru_gen_folio belongs to */ u8 gen; /* the list segment this lru_gen_folio belongs to */ u8 seg; /* per-node lru_gen_folio list for global reclaim */ struct hlist_nulls_node list;};
enum { MM_LEAF_TOTAL, /* total leaf entries */ MM_LEAF_YOUNG, /* young leaf entries */ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ NR_MM_STATS};
/* double-buffering Bloom filters */#define NR_BLOOM_FILTERS 2
struct lru_gen_mm_state { /* synced with max_seq after each iteration */ unsigned long seq; /* where the current iteration continues after */ struct list_head *head; /* where the last iteration ended before */ struct list_head *tail; /* Bloom filters flip after each iteration */ unsigned long *filters[NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS];};
struct lru_gen_mm_walk { /* the lruvec under reclaim */ struct lruvec *lruvec; /* max_seq from lru_gen_folio: can be out of date */ unsigned long seq; /* the next address within an mm to scan */ unsigned long next_addr; /* to batch promoted pages */ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* to batch the mm stats */ int mm_stats[NR_MM_STATS]; /* total batched items */ int batched; int swappiness; bool force_scan;};
/* * For each node, memcgs are divided into two generations: the old and the * young. For each generation, memcgs are randomly sharded into multiple bins * to improve scalability. For each bin, the hlist_nulls is virtually divided * into two segments: the tail and the default. * * An onlining memcg is added to the tail of a random bin in the old generation. * The eviction starts at the head of a random bin in the old generation. The * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes * the old generation, is incremented when all its bins become empty. * * There are three operations: * 1. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its * current generation (old or young) and updates its "seg" to "tail"; * 2. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old * generation, updates its "gen" to "old" and resets its "seg" to "default"; * 3. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the * young generation, updates its "gen" to "young" and resets its "seg" to * "default". * * The events that trigger the above operations are: * 1. The first attempt to reclaim a memcg below low, which triggers * MEMCG_LRU_TAIL; * 2. The first attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_TAIL; * 3. The second attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_YOUNG; * 4. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; * 5. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; * 6. Offlining a memcg, which triggers MEMCG_LRU_OLD. * * Notes: * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing * of their max_seq counters ensures the eventual fairness to all eligible * memcgs. For memcg reclaim, it still relies on mem_cgroup_iter(). * 2. There are only two valid generations: old (seq) and young (seq+1). * MEMCG_NR_GENS is set to three so that when reading the generation counter * locklessly, a stale value (seq-1) does not wraparound to young. */#define MEMCG_NR_GENS 3#define MEMCG_NR_BINS 8
struct lru_gen_memcg { /* the per-node memcg generation counter */ unsigned long seq; /* each memcg has one lru_gen_folio per node */ unsigned long nr_memcgs[MEMCG_NR_GENS]; /* per-node lru_gen_folio list for global reclaim */ struct hlist_nulls_head fifo[MEMCG_NR_GENS][MEMCG_NR_BINS]; /* protects the above */ spinlock_t lock;};
void lru_gen_init_pgdat(struct pglist_data *pgdat);void lru_gen_init_lruvec(struct lruvec *lruvec);bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
void lru_gen_init_memcg(struct mem_cgroup *memcg);void lru_gen_exit_memcg(struct mem_cgroup *memcg);void lru_gen_online_memcg(struct mem_cgroup *memcg);void lru_gen_offline_memcg(struct mem_cgroup *memcg);void lru_gen_release_memcg(struct mem_cgroup *memcg);void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid);bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid);void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
#else /* !CONFIG_LRU_GEN */
static inline void lru_gen_init_pgdat(struct pglist_data *pgdat){}
static inline void lru_gen_init_lruvec(struct lruvec *lruvec){}
static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr){ return false;}
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_online_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_offline_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_release_memcg(struct mem_cgroup *memcg){}
static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid){}
static inline bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid){ return true;}
static inlinevoid lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){}
#endif /* CONFIG_LRU_GEN */
struct lru_cost { unsigned long count; unsigned long last_rotated; unsigned long last_io;};
struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ spinlock_t lru_lock; /* * These track the cost of reclaiming one LRU - file or anon - * over the other. As the observed cost of reclaiming one LRU * increases, the reclaim scan balance tips toward the other. * Updated and decayed at prepare_scan_control() time; cost_lock * serialises that update. */ struct lru_cost cost[ANON_AND_FILE]; /* Protects cost[]. */ spinlock_t cost_lock; /* Non-resident age, driven by LRU movement */ atomic_long_t nonresident_age; /* Refaults at the time of last reclaim cycle */ unsigned long refaults[ANON_AND_FILE]; /* Various lruvec state flags (enum lruvec_flags) */ unsigned long flags;#ifdef CONFIG_LRU_GEN /* evictable pages divided into generations */ struct lru_gen_folio lrugen;#ifdef CONFIG_LRU_GEN_WALKS_MMU /* to concurrently iterate lru_gen_mm_list */ struct lru_gen_mm_state mm_state;#endif#endif /* CONFIG_LRU_GEN */#ifdef CONFIG_MEMCG struct pglist_data *pgdat;#endif struct zswap_lruvec_state zswap_lruvec_state;};
/* Isolate for asynchronous migration */#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)/* Isolate unevictable pages */#define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
/* LRU Isolation modes. */typedef unsigned __bitwise isolate_mode_t;
enum zone_watermarks { WMARK_MIN, WMARK_LOW, WMARK_HIGH, WMARK_PROMO, NR_WMARK};
/* * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. Two additional lists * are added for THP. One PCP list is used by GPF_MOVABLE, and the other PCP list * is used by GFP_UNMOVABLE and GFP_RECLAIMABLE. */#ifdef CONFIG_TRANSPARENT_HUGEPAGE#define NR_PCP_THP 2#else#define NR_PCP_THP 0#endif#define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1))#define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP)
/* * Flags used in pcp->flags field. * * PCPF_PREV_FREE_HIGH_ORDER: a high-order page is freed in the * previous page freeing. To avoid to drain PCP for an accident * high-order page freeing. * * PCPF_FREE_HIGH_BATCH: preserve "pcp->batch" pages in PCP before * draining PCP for consecutive high-order pages freeing without * allocation if data cache slice of CPU is large enough. To reduce * zone lock contention and keep cache-hot pages reusing. */#define PCPF_PREV_FREE_HIGH_ORDER BIT(0)#define PCPF_FREE_HIGH_BATCH BIT(1)
struct per_cpu_pages { spinlock_t lock; /* Protects lists field */ int count; /* number of pages in the list */ int high; /* high watermark, emptying needed */ int high_min; /* min high watermark */ int high_max; /* max high watermark */ int batch; /* chunk size for buddy add/remove */ u8 flags; /* protected by pcp->lock */ u8 alloc_factor; /* batch scaling factor during allocate */#ifdef CONFIG_NUMA u8 expire; /* When 0, remote pagesets are drained */#endif short free_count; /* consecutive free count */
/* Lists of pages, one per migrate type stored on the pcp-lists */ struct list_head lists[NR_PCP_LISTS];} ____cacheline_aligned_in_smp;
struct per_cpu_zonestat {#ifdef CONFIG_SMP s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; s8 stat_threshold;#endif#ifdef CONFIG_NUMA /* * Low priority inaccurate counters that are only folded * on demand. Use a large type to avoid the overhead of * folding during refresh_cpu_vm_stats. */ unsigned long vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];#endif};
struct per_cpu_nodestat { s8 stat_threshold; s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];};
#endif /* !__GENERATING_BOUNDS.H */
enum zone_type { /* * ZONE_DMA and ZONE_DMA32 are used when there are peripherals not able * to DMA to all of the addressable memory (ZONE_NORMAL). * On architectures where this area covers the whole 32 bit address * space ZONE_DMA32 is used. ZONE_DMA is left for the ones with smaller * DMA addressing constraints. This distinction is important as a 32bit * DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit * platforms may need both zones as they support peripherals with * different DMA addressing limitations. */#ifdef CONFIG_ZONE_DMA ZONE_DMA,#endif#ifdef CONFIG_ZONE_DMA32 ZONE_DMA32,#endif /* * Normal addressable memory is in ZONE_NORMAL. DMA operations can be * performed on pages in ZONE_NORMAL if the DMA devices support * transfers to all addressable memory. */ ZONE_NORMAL,#ifdef CONFIG_HIGHMEM /* * A memory area that is only addressable by the kernel through * mapping portions into its own address space. This is for example * used by i386 to allow the kernel to address the memory beyond * 900MB. The kernel will set up special mappings (page * table entries on i386) for each page that the kernel needs to * access. */ ZONE_HIGHMEM,#endif /* * ZONE_MOVABLE is similar to ZONE_NORMAL, except that it contains * movable pages with few exceptional cases described below. Main use * cases for ZONE_MOVABLE are to make memory offlining/unplug more * likely to succeed, and to locally limit unmovable allocations - e.g., * to increase the number of THP/huge pages. Notable special cases are: * * 1. Pinned pages: (long-term) pinning of movable pages might * essentially turn such pages unmovable. Therefore, we do not allow * pinning long-term pages in ZONE_MOVABLE. When pages are pinned and * faulted, they come from the right zone right away. However, it is * still possible that address space already has pages in * ZONE_MOVABLE at the time when pages are pinned (i.e. user has * touches that memory before pinning). In such case we migrate them * to a different zone. When migration fails - pinning fails. * 2. memblock allocations: kernelcore/movablecore setups might create * situations where ZONE_MOVABLE contains unmovable allocations * after boot. Memory offlining and allocations fail early. * 3. Memory holes: kernelcore/movablecore setups might create very rare * situations where ZONE_MOVABLE contains memory holes after boot, * for example, if we have sections that are only partially * populated. Memory offlining and allocations fail early. * 4. PG_hwpoison pages: while poisoned pages can be skipped during * memory offlining, such pages cannot be allocated. * 5. Unmovable PG_offline pages: in paravirtualized environments, * hotplugged memory blocks might only partially be managed by the * buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The * parts not manged by the buddy are unmovable PG_offline pages. In * some cases (virtio-mem), such pages can be skipped during * memory offlining, however, cannot be moved/allocated. These * techniques might use alloc_contig_range() to hide previously * exposed pages from the buddy again (e.g., to implement some sort * of memory unplug in virtio-mem). * 6. ZERO_PAGE(0), kernelcore/movablecore setups might create * situations where ZERO_PAGE(0) which is allocated differently * on different platforms may end up in a movable zone. ZERO_PAGE(0) * cannot be migrated. * 7. Memory-hotplug: when using memmap_on_memory and onlining the * memory to the MOVABLE zone, the vmemmap pages are also placed in * such zone. Such pages cannot be really moved around as they are * self-stored in the range, but they are treated as movable when * the range they describe is about to be offlined. * * In general, no unmovable allocations that degrade memory offlining * should end up in ZONE_MOVABLE. Allocators (like alloc_contig_range()) * have to expect that migrating pages in ZONE_MOVABLE can fail (even * if has_unmovable_pages() states that there are no unmovable pages, * there can be false negatives). */ ZONE_MOVABLE,#ifdef CONFIG_ZONE_DEVICE ZONE_DEVICE,#endif __MAX_NR_ZONES
};
#ifndef __GENERATING_BOUNDS_H
#define ASYNC_AND_SYNC 2
struct zone { /* Read-mostly fields */
/* zone watermarks, access with *_wmark_pages(zone) macros */ unsigned long _watermark[NR_WMARK]; unsigned long watermark_boost;
unsigned long nr_reserved_highatomic; unsigned long nr_free_highatomic;
/* * We don't know if the memory that we're going to allocate will be * freeable or/and it will be released eventually, so to avoid totally * wasting several GB of ram we must reserve some of the lower zone * memory (otherwise we risk to run OOM on the lower zones despite * there being tons of freeable ram on the higher zones). This array is * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl * changes. */ long lowmem_reserve[MAX_NR_ZONES];
#ifdef CONFIG_NUMA int node;#endif struct pglist_data *zone_pgdat; struct per_cpu_pages __percpu *per_cpu_pageset; struct per_cpu_zonestat __percpu *per_cpu_zonestats; /* * the high and batch values are copied to individual pagesets for * faster access */ int pageset_high_min; int pageset_high_max; int pageset_batch;
#ifndef CONFIG_SPARSEMEM /* * Flags for a pageblock_nr_pages block. See pageblock-flags.h. * In SPARSEMEM, this map is stored in struct mem_section */ unsigned long *pageblock_flags;#endif /* CONFIG_SPARSEMEM */
/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ unsigned long zone_start_pfn;
/* * spanned_pages is the total pages spanned by the zone, including * holes, which is calculated as: * spanned_pages = zone_end_pfn - zone_start_pfn; * * present_pages is physical pages existing within the zone, which * is calculated as: * present_pages = spanned_pages - absent_pages(pages in holes); * * present_early_pages is present pages existing within the zone * located on memory available since early boot, excluding hotplugged * memory. * * managed_pages is present pages managed by the buddy system, which * is calculated as (reserved_pages includes pages allocated by the * bootmem allocator): * managed_pages = present_pages - reserved_pages; * * cma pages is present pages that are assigned for CMA use * (MIGRATE_CMA). * * pages_with_online_memmap tracks pages within the zone that have * an online memory map: present pages and memory holes whose * memory map has been initialized and pfn_to_online_page() * succeeds. When spanned_pages == pages_with_online_memmap, * pfn_to_page() can be performed without further checks on any * PFN within the zone span. * * Note: this counter may temporarily undercount when pages with an * online memory map exist outside the current zone span. Such pages * are only created during boot, when initializing the memory map of * pages that do not fall into any zone span. The undercount itself * can only happen after boot, during memory hotplug, when growing * the zone to cover such pages and later shrinking it back, which * may result in a "too small" value. This is safe: it merely * prevents detecting a contiguous zone. * * Here is an example (page numbers are just for illustration * purposes): * after boot: * [ zone span ] * [ zone pages ] * spanned=10, initialized=15, online=10 * online == spanned -> contiguous * * growing after hotplug (hotplug 5): * [ zone span ] * [ zone pages ] [ zone pages ] * spanned=30, initialized=20, online=15 * online != spanned -> not contiguous * * shrinking after hotunplug (hotunplug 5 again): * [ zone span ] * [ zone pages ] * spanned=15, initialized=15, online=10 * online != spanned -> not contiguous although contiguous * * So present_pages may be used by memory hotplug or memory power * management logic to figure out unmanaged pages by checking * (present_pages - managed_pages). And managed_pages should be used * by page allocator and vm scanner to calculate all kinds of watermarks * and thresholds. * * Locking rules: * * zone_start_pfn and spanned_pages are protected by span_seqlock. * It is a seqlock because it has to be read outside of zone->lock, * and it is done in the main allocator path. But, it is written * quite infrequently. * * The span_seq lock is declared along with zone->lock because it is * frequently read in proximity to zone->lock. It's good to * give them a chance of being in the same cacheline. * * Write access to present_pages at runtime should be protected by * mem_hotplug_begin/done(). Any reader who can't tolerant drift of * present_pages should use get_online_mems() to get a stable value. */ atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages; unsigned long pages_with_online_memmap;#if defined(CONFIG_MEMORY_HOTPLUG) unsigned long present_early_pages;#endif#ifdef CONFIG_CMA unsigned long cma_pages;#endif
const char *name;
#ifdef CONFIG_MEMORY_ISOLATION /* * Number of isolated pageblock. It is used to solve incorrect * freepage counting problem due to racy retrieving migratetype * of pageblock. Protected by zone->lock. */ unsigned long nr_isolate_pageblock;#endif
#ifdef CONFIG_MEMORY_HOTPLUG /* see spanned/present_pages for more description */ seqlock_t span_seqlock;#endif
int initialized;
/* Write-intensive fields used from the page allocator */ CACHELINE_PADDING(_pad1_);
/* free areas of different sizes */ struct free_area free_area[NR_PAGE_ORDERS];
#ifdef CONFIG_UNACCEPTED_MEMORY /* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */ struct list_head unaccepted_pages;#endif
/* zone flags, see below */ unsigned long flags;
/* Primarily protects free_area */ spinlock_t lock;
/* Pages to be freed when next trylock succeeds */ struct llist_head trylock_free_pages;
/* Write-intensive fields used by compaction and vmstats. */ CACHELINE_PADDING(_pad2_);
/* * When free pages are below this point, additional steps are taken * when reading the number of free pages to avoid per-cpu counter * drift allowing watermarks to be breached */ unsigned long percpu_drift_mark;
#if defined CONFIG_COMPACTION || defined CONFIG_CMA /* pfn where compaction free scanner should start */ unsigned long compact_cached_free_pfn; /* pfn where compaction migration scanner should start */ unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC]; unsigned long compact_init_migrate_pfn; unsigned long compact_init_free_pfn;#endif
#ifdef CONFIG_COMPACTION /* * On compaction failure, 1<<compact_defer_shift compactions * are skipped before trying again. The number attempted since * last failure is tracked with compact_considered. * compact_order_failed is the minimum compaction failed order. */ unsigned int compact_considered; unsigned int compact_defer_shift; int compact_order_failed;#endif
#if defined CONFIG_COMPACTION || defined CONFIG_CMA /* Set to true when the PG_migrate_skip bits should be cleared */ bool compact_blockskip_flush;#endif
bool contiguous;
CACHELINE_PADDING(_pad3_); /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];#ifdef CONFIG_VMEMMAP_OPTIMIZATION struct page **vmemmap_tails;#endif} ____cacheline_internodealigned_in_smp;
enum pgdat_flags { PGDAT_WRITEBACK, /* reclaim scanning has recently found * many pages under writeback */ PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */};
enum zone_flags { ZONE_BOOSTED_WATERMARK, /* zone recently boosted watermarks. * Cleared when kswapd is woken. */ ZONE_RECLAIM_ACTIVE, /* kswapd may be scanning the zone. */ ZONE_BELOW_HIGH, /* zone is below high watermark. */};
static inline unsigned long wmark_pages(const struct zone *z, enum zone_watermarks w){ return z->_watermark[w] + z->watermark_boost;}
static inline unsigned long min_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_MIN);}
static inline unsigned long low_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_LOW);}
static inline unsigned long high_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_HIGH);}
static inline unsigned long promo_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_PROMO);}
static inline unsigned long zone_managed_pages(const struct zone *zone){ return (unsigned long)atomic_long_read(&zone->managed_pages);}
static inline unsigned long zone_cma_pages(struct zone *zone){#ifdef CONFIG_CMA return zone->cma_pages;#else return 0;#endif}
static inline unsigned long zone_end_pfn(const struct zone *zone){ return zone->zone_start_pfn + zone->spanned_pages;}
static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn){ return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);}
static inline bool zone_is_initialized(const struct zone *zone){ return zone->initialized;}
static inline bool zone_is_empty(const struct zone *zone){ return zone->spanned_pages == 0;}
#ifndef BUILD_VDSO32_64/* * The zone field is never updated after free_area_init_core() * sets it, so none of the operations on it need to be atomic. */
/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH)#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH)
/* * Define the bit shifts to access each section. For non-existent * sections we define the shift as 0; that plus a 0 mask ensures * the compiler will optimise away reference to them. */#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))#define KASAN_TAG_PGSHIFT (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0))
/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */#ifdef NODE_NOT_IN_PAGE_FLAGS#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF) ? \ SECTIONS_PGOFF : ZONES_PGOFF)#else#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF) ? \ NODES_PGOFF : ZONES_PGOFF)#endif
#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)#define NODES_MASK ((1UL << NODES_WIDTH) - 1)#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)#define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1)#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags){#if ZONES_WIDTH != 0 ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT);#endif return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK;}
static inline enum zone_type page_zonenum(const struct page *page){ return memdesc_zonenum(&page->flags);}
static inline enum zone_type folio_zonenum(const struct folio *folio){ return memdesc_zonenum(&folio->flags);}
#ifdef CONFIG_ZONE_DEVICEstatic inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf){ return memdesc_zonenum(mdf) == ZONE_DEVICE;}
static inline struct dev_pagemap *page_pgmap(const struct page *page){ VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap;}
/* * Consecutive zone device pages should not be merged into the same sgl * or bvec segment with other types of pages or if they belong to different * pgmaps. Otherwise getting the pgmap of a given segment is not possible * without scanning the entire segment. This helper returns true either if * both pages are not zone device pages or both pages are zone device pages * with the same pgmap. */static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b){ if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b);}
extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *);#elsestatic inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf){ return false;}static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b){ return true;}static inline struct dev_pagemap *page_pgmap(const struct page *page){ return NULL;}#endif
static inline bool is_zone_device_page(const struct page *page){ return memdesc_is_zone_device(&page->flags);}
static inline bool folio_is_zone_device(const struct folio *folio){ return memdesc_is_zone_device(&folio->flags);}
static inline bool is_zone_movable_page(const struct page *page){ return page_zonenum(page) == ZONE_MOVABLE;}
static inline bool folio_is_zone_movable(const struct folio *folio){ return folio_zonenum(folio) == ZONE_MOVABLE;}#endif
/* * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty * intersection with the given zone */static inline bool zone_intersects(const struct zone *zone, unsigned long start_pfn, unsigned long nr_pages){ if (zone_is_empty(zone)) return false; if (start_pfn >= zone_end_pfn(zone) || start_pfn + nr_pages <= zone->zone_start_pfn) return false;
return true;}
/* * The "priority" of VM scanning is how much of the queues we will scan in one * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the * queues ("queue_length >> 12") during an aging round. */#define DEF_PRIORITY 12
/* Maximum number of zones on a zonelist */#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
enum { ZONELIST_FALLBACK, /* zonelist with fallback */#ifdef CONFIG_NUMA /* * The NUMA zonelists are doubled because we need zonelists that * restrict the allocations to a single node for __GFP_THISNODE. */ ZONELIST_NOFALLBACK, /* zonelist without fallback (__GFP_THISNODE) */#endif MAX_ZONELISTS};
/* * This struct contains information about a zone in a zonelist. It is stored * here to avoid dereferences into large structures and lookups of tables */struct zoneref { struct zone *zone; /* Pointer to actual zone */ int zone_idx; /* zone_idx(zoneref->zone) */};
/* * One allocation request operates on a zonelist. A zonelist * is a list of zones, the first one is the 'goal' of the * allocation, the other zones are fallback zones, in decreasing * priority. * * To speed the reading of the zonelist, the zonerefs contain the zone index * of the entry being read. Helper functions to access information given * a struct zoneref are * * zonelist_zone() - Return the struct zone * for an entry in _zonerefs * zonelist_zone_idx() - Return the index of the zone for an entry * zonelist_node_idx() - Return the index of the node for an entry */struct zonelist { struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];};
/* * The array of struct pages for flatmem. * It must be declared for SPARSEMEM as well because there are configurations * that rely on that. */extern struct page *mem_map;
#ifdef CONFIG_MEMORY_FAILURE/* * Per NUMA node memory failure handling statistics. */struct memory_failure_stats { /* * Number of raw pages poisoned. * Cases not accounted: memory outside kernel control, offline page, * arch-specific memory_failure (SGX), hwpoison_filter() filtered * error events, and unpoison actions from hwpoison_unpoison. */ unsigned long total; /* * Recovery results of poisoned raw pages handled by memory_failure, * in sync with mf_result. * total = ignored + failed + delayed + recovered. * total * PAGE_SIZE * #nodes = /proc/meminfo/HardwareCorrupted. */ unsigned long ignored; unsigned long failed; unsigned long delayed; unsigned long recovered;};#endif
/* * On NUMA machines, each NUMA node would have a pg_data_t to describe * it's memory layout. On UMA machines there is a single pglist_data which * describes the whole memory. * * Memory statistics and page replacement data structures are maintained on a * per-zone basis. */typedef struct pglist_data { /* * node_zones contains just the zones for THIS node. Not all of the * zones may be populated, but it is the full list. It is referenced by * this node's node_zonelists as well as other node's node_zonelists. */ struct zone node_zones[MAX_NR_ZONES];
/* * node_zonelists contains references to all zones in all nodes. * Generally the first zones will be references to this node's * node_zones. */ struct zonelist node_zonelists[MAX_ZONELISTS];
int nr_zones; /* number of populated zones in this node */#ifdef CONFIG_FLATMEM /* means !SPARSEMEM */ struct page *node_mem_map;#ifdef CONFIG_PAGE_EXTENSION struct page_ext *node_page_ext;#endif#endif#if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) /* * Must be held any time you expect node_start_pfn, * node_present_pages, node_spanned_pages or nr_zones to stay constant. * Also synchronizes pgdat->first_deferred_pfn during deferred page * init. * * pgdat_resize_lock() and pgdat_resize_unlock() are provided to * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG * or CONFIG_DEFERRED_STRUCT_PAGE_INIT. * * Nests above zone->lock and zone->span_seqlock */ spinlock_t node_size_lock;#endif unsigned long node_start_pfn; unsigned long node_present_pages; /* total number of physical pages */ unsigned long node_spanned_pages; /* total size of physical page range, including holes */ int node_id; wait_queue_head_t kswapd_wait; wait_queue_head_t pfmemalloc_wait;
/* workqueues for throttling reclaim for different reasons. */ wait_queue_head_t reclaim_wait[NR_VMSCAN_THROTTLE];
atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */ unsigned long nr_reclaim_start; /* nr pages written while throttled * when throttling started. */#ifdef CONFIG_MEMORY_HOTPLUG struct mutex kswapd_lock;#endif struct task_struct *kswapd; /* Protected by kswapd_lock */ int kswapd_order; enum zone_type kswapd_highest_zoneidx;
atomic_t kswapd_failures; /* Number of 'reclaimed == 0' runs */
#ifdef CONFIG_COMPACTION int kcompactd_max_order; enum zone_type kcompactd_highest_zoneidx; wait_queue_head_t kcompactd_wait; struct task_struct *kcompactd; bool proactive_compact_trigger;#endif /* * This is a per-node reserve of pages that are not available * to userspace allocations. */ unsigned long totalreserve_pages;
#ifdef CONFIG_NUMA /* * node reclaim becomes active if more unmapped pages exist. */ unsigned long min_unmapped_pages; unsigned long min_slab_pages;#endif /* CONFIG_NUMA */
/* Write-intensive fields used by page reclaim */ CACHELINE_PADDING(_pad1_);
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* * If memory initialisation on large machines is deferred then this * is the first PFN that needs to be initialised. */ unsigned long first_deferred_pfn;#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
#ifdef CONFIG_NUMA_BALANCING /* start time in ms of current promote rate limit period */ unsigned int nbp_rl_start; /* number of promote candidate pages at start time of current rate limit period */ unsigned long nbp_rl_nr_cand; /* promote threshold in ms */ unsigned int nbp_threshold; /* start time in ms of current promote threshold adjustment period */ unsigned int nbp_th_start; /* * number of promote candidate pages at start time of current promote * threshold adjustment period */ unsigned long nbp_th_nr_cand;#endif /* Fields commonly accessed by the page reclaim scanner */
/* * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED. * * Use mem_cgroup_lruvec() to look up lruvecs. */ struct lruvec __lruvec;
unsigned long flags;
#ifdef CONFIG_LRU_GEN /* kswap mm walk data */ struct lru_gen_mm_walk mm_walk; /* lru_gen_folio list */ struct lru_gen_memcg memcg_lru;#endif
CACHELINE_PADDING(_pad2_);
/* Per-node vmstats */ struct per_cpu_nodestat __percpu *per_cpu_nodestats; atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];#ifdef CONFIG_NUMA struct memory_tier __rcu *memtier;#endif#ifdef CONFIG_MEMORY_FAILURE struct memory_failure_stats mf_stats;#endif} pg_data_t;
#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat){ return pgdat->node_start_pfn + pgdat->node_spanned_pages;}
#include <linux/memory_hotplug.h>
void build_all_zonelists(pg_data_t *pgdat);bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, int highest_zoneidx, unsigned int alloc_flags, long free_pages);bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, int highest_zoneidx, unsigned int alloc_flags);
enum kswapd_clear_hopeless_reason { KSWAPD_CLEAR_HOPELESS_OTHER = 0, KSWAPD_CLEAR_HOPELESS_KSWAPD, KSWAPD_CLEAR_HOPELESS_DIRECT, KSWAPD_CLEAR_HOPELESS_PCP,};
void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order, enum zone_type highest_zoneidx);void kswapd_try_clear_hopeless(struct pglist_data *pgdat, unsigned int order, int highest_zoneidx);void kswapd_clear_hopeless(pg_data_t *pgdat, enum kswapd_clear_hopeless_reason reason);bool kswapd_test_hopeless(pg_data_t *pgdat);
/* * Memory initialization context, use to differentiate memory added by * the platform statically or via memory hotplug interface. */enum meminit_context { MEMINIT_EARLY, MEMINIT_HOTPLUG,};
extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn, unsigned long size);
extern void lruvec_init(struct lruvec *lruvec);
static inline struct pglist_data *lruvec_pgdat(const struct lruvec *lruvec){#ifdef CONFIG_MEMCG return lruvec->pgdat;#else return container_of(lruvec, struct pglist_data, __lruvec);#endif}
#ifdef CONFIG_HAVE_MEMORYLESS_NODESint local_memory_node(int node_id);#elsestatic inline int local_memory_node(int node_id) { return node_id; };#endif
/* * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc. */#define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
#ifdef CONFIG_ZONE_DEVICEstatic inline bool zone_is_zone_device(const struct zone *zone){ return zone_idx(zone) == ZONE_DEVICE;}#elsestatic inline bool zone_is_zone_device(const struct zone *zone){ return false;}#endif
/** * zone_is_contiguous - test whether a zone is contiguous * @zone: the zone to test. * * In a contiguous zone, it is valid to call pfn_to_page() on any PFN in the * spanned zone without requiring pfn_valid() or pfn_to_online_page() checks. * * Note that missing synchronization with memory offlining makes any PFN * traversal prone to races. * * ZONE_DEVICE zones are always marked non-contiguous. * * Return: true if contiguous, otherwise false. */static inline bool zone_is_contiguous(const struct zone *zone){ return READ_ONCE(zone->contiguous);}
static inline void set_zone_contiguous(struct zone *zone){ if (zone_is_zone_device(zone)) return; if (zone->spanned_pages == zone->pages_with_online_memmap) WRITE_ONCE(zone->contiguous, true);}
static inline void clear_zone_contiguous(struct zone *zone){ WRITE_ONCE(zone->contiguous, false);}
/* * Returns true if a zone has pages managed by the buddy allocator. * All the reclaim decisions have to use this function rather than * populated_zone(). If the whole zone is reserved then we can easily * end up with populated_zone() && !managed_zone(). */static inline bool managed_zone(const struct zone *zone){ return zone_managed_pages(zone);}
/* Returns true if a zone has memory */static inline bool populated_zone(const struct zone *zone){ return zone->present_pages;}
#ifdef CONFIG_NUMAstatic inline int zone_to_nid(const struct zone *zone){ return zone->node;}
static inline void zone_set_nid(struct zone *zone, int nid){ zone->node = nid;}#elsestatic inline int zone_to_nid(const struct zone *zone){ return 0;}
static inline void zone_set_nid(struct zone *zone, int nid) {}#endif
extern int movable_zone;
static inline int is_highmem_idx(enum zone_type idx){#ifdef CONFIG_HIGHMEM return (idx == ZONE_HIGHMEM || (idx == ZONE_MOVABLE && movable_zone == ZONE_HIGHMEM));#else return 0;#endif}
/** * is_highmem - helper function to quickly check if a struct zone is a * highmem zone or not. This is an attempt to keep references * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum. * @zone: pointer to struct zone variable * Return: 1 for a highmem zone, 0 otherwise */static inline int is_highmem(const struct zone *zone){ return is_highmem_idx(zone_idx(zone));}
bool has_managed_zone(enum zone_type zone);static inline bool has_managed_dma(void){#ifdef CONFIG_ZONE_DMA return has_managed_zone(ZONE_DMA);#else return false;#endif}
#ifndef CONFIG_NUMA
extern struct pglist_data contig_page_data;static inline struct pglist_data *NODE_DATA(int nid){ return &contig_page_data;}
#else /* CONFIG_NUMA */
#include <asm/mmzone.h>
#endif /* !CONFIG_NUMA */
extern struct pglist_data *first_online_pgdat(void);extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);extern struct zone *next_zone(struct zone *zone);
/** * for_each_online_pgdat - helper macro to iterate over all online nodes * @pgdat: pointer to a pg_data_t variable */#define for_each_online_pgdat(pgdat) \ for (pgdat = first_online_pgdat(); \ pgdat; \ pgdat = next_online_pgdat(pgdat))/** * for_each_zone - helper macro to iterate over all memory zones * @zone: pointer to struct zone variable * * The user only needs to declare the zone variable, for_each_zone * fills it in. */#define for_each_zone(zone) \ for (zone = (first_online_pgdat())->node_zones; \ zone; \ zone = next_zone(zone))
#define for_each_populated_zone(zone) \ for (zone = (first_online_pgdat())->node_zones; \ zone; \ zone = next_zone(zone)) \ if (!populated_zone(zone)) \ ; /* do nothing */ \ else
static inline struct zone *zonelist_zone(struct zoneref *zoneref){ return zoneref->zone;}
static inline int zonelist_zone_idx(const struct zoneref *zoneref){ return zoneref->zone_idx;}
static inline int zonelist_node_idx(const struct zoneref *zoneref){ return zone_to_nid(zoneref->zone);}
struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, const nodemask_t *nodes);
/** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point * @z: The cursor used as a starting point for the search * @highest_zoneidx: The zone index of the highest zone to return * @nodes: An optional nodemask to filter the zonelist with * * This function returns the next zone at or below a given zone index that is * within the allowed nodemask using a cursor as the starting point for the * search. The zoneref returned is a cursor that represents the current zone * being examined. It should be advanced by one before calling * next_zones_zonelist again. * * Return: the next zone at or below highest_zoneidx within the allowed * nodemask using a cursor within a zonelist as a starting point */static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, const nodemask_t *nodes){ if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; return __next_zones_zonelist(z, highest_zoneidx, nodes);}
/** * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist * @zonelist: The zonelist to search for a suitable zone * @highest_zoneidx: The zone index of the highest zone to return * @nodes: An optional nodemask to filter the zonelist with * * This function returns the first zone at or below a given zone index that is * within the allowed nodemask. The zoneref returned is a cursor that can be * used to iterate the zonelist with next_zones_zonelist by advancing it by * one before calling. * * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is * never NULL). This may happen either genuinely, or due to concurrent nodemask * update due to cpuset modification. * * Return: Zoneref pointer for the first suitable zone found */static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, const nodemask_t *nodes){ return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes);}
/** * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask * @zone: The current zone in the iterator * @z: The current pointer within zonelist->_zonerefs being iterated * @zlist: The zonelist being iterated * @highidx: The zone index of the highest zone to return * @nodemask: Nodemask allowed by the allocator * * This iterator iterates though all zones at or below a given zone index and * within a given nodemask */#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \ for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \ zone; \ z = next_zones_zonelist(++z, highidx, nodemask), \ zone = zonelist_zone(z))
#define for_next_zone_zonelist_nodemask(zone, z, highidx, nodemask) \ for (zone = zonelist_zone(z); \ zone; \ z = next_zones_zonelist(++z, highidx, nodemask), \ zone = zonelist_zone(z))
/** * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index * @zone: The current zone in the iterator * @z: The current pointer within zonelist->zones being iterated * @zlist: The zonelist being iterated * @highidx: The zone index of the highest zone to return * * This iterator iterates though all zones at or below a given zone index. */#define for_each_zone_zonelist(zone, z, zlist, highidx) \ for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
/* Whether the 'nodes' are all movable nodes */static inline bool movable_only_nodes(nodemask_t *nodes){ struct zonelist *zonelist; struct zoneref *z; int nid;
if (nodes_empty(*nodes)) return false;
/* * We can chose arbitrary node from the nodemask to get a * zonelist as they are interlinked. We just need to find * at least one zone that can satisfy kernel allocations. */ nid = first_node(*nodes); zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK]; z = first_zones_zonelist(zonelist, ZONE_NORMAL, nodes); return (!zonelist_zone(z)) ? true : false;}
#ifdef CONFIG_SPARSEMEM#include <asm/sparsemem.h>#endif
#ifdef CONFIG_FLATMEM#define pfn_to_nid(pfn) (0)#endif
#ifdef CONFIG_SPARSEMEM
/* * PA_SECTION_SHIFT physical address to/from section number * PFN_SECTION_SHIFT pfn to/from section number */#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
#define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)#define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
#define SECTION_BLOCKFLAGS_BITS \ ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
#if (MAX_PAGE_ORDER + PAGE_SHIFT) > SECTION_SIZE_BITS#error Allocator MAX_PAGE_ORDER exceeds SECTION_SIZE#endif
static inline unsigned long pfn_to_section_nr(unsigned long pfn){ return pfn >> PFN_SECTION_SHIFT;}static inline unsigned long section_nr_to_pfn(unsigned long sec){ return sec << PFN_SECTION_SHIFT;}
#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
#define SUBSECTION_SHIFT 21#define SUBSECTION_SIZE (1UL << SUBSECTION_SHIFT)
#define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)#define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT)#define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
#if SUBSECTION_SHIFT > SECTION_SIZE_BITS#error Subsection size exceeds section size#else#define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))#endif
#define SUBSECTION_ALIGN_UP(pfn) ALIGN((pfn), PAGES_PER_SUBSECTION)#define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
struct mem_section_usage { struct rcu_head rcu;#ifdef CONFIG_SPARSEMEM_VMEMMAP DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);#endif /* See declaration of similar field in struct zone */ unsigned long pageblock_flags[0];};
struct page;struct page_ext;struct mem_section { /* * This is, logically, a pointer to an array of struct * pages. However, it is stored with some other magic. * (see sparse_init_one_section()) * * Additionally during early boot we encode node id of * the location of the section here to guide allocation. * (see sparse.c::memory_present()) * * Making it a UL at least makes someone do a cast * before using it wrong. */ unsigned long section_mem_map;
struct mem_section_usage *usage;#ifdef CONFIG_VMEMMAP_OPTIMIZATION /* * Normally, sections hold regular (order-0) pages. However, for * sections with HVO enabled, this tracks the compound page order * to enable deduplication of redundant vmemmap pages. */ unsigned int compound_page_order;#endif#ifdef CONFIG_PAGE_EXTENSION /* * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use * section. (see page_ext.h about this.) */ struct page_ext *page_ext;#endif/* Sacrifice minor padding space for efficient lookup. */} __aligned(2 * sizeof(unsigned long));
#ifdef CONFIG_SPARSEMEM_EXTREME#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))#else#define SECTIONS_PER_ROOT 1#endif
#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)#define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
#ifdef CONFIG_SPARSEMEM_EXTREMEextern struct mem_section **mem_section;#elseextern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];#endif
static inline unsigned long *section_to_usemap(struct mem_section *ms){ return ms->usage->pageblock_flags;}
static inline struct mem_section *__nr_to_section(unsigned long nr){ unsigned long root = SECTION_NR_TO_ROOT(nr);
if (unlikely(root >= NR_SECTION_ROOTS)) return NULL;
#ifdef CONFIG_SPARSEMEM_EXTREME if (!mem_section || !mem_section[root]) return NULL;#endif return &mem_section[root][nr % SECTIONS_PER_ROOT];}
/* * We use the lower bits of the mem_map pointer to store a little bit of * information. The pointer is calculated as mem_map - section_nr_to_pfn(). * The result is aligned to the minimum alignment of the two values: * * 1. All mem_map arrays are page-aligned. * 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT lowest bits. * * We always expect a single section to cover full pages. Therefore, * we can safely assume that PFN_SECTION_SHIFT is large enough to * accommodate SECTION_MAP_LAST_BIT. We use BUILD_BUG_ON() to ensure this. */enum { SECTION_HAS_MEM_MAP_BIT, SECTION_IS_ONLINE_BIT, SECTION_IS_EARLY_BIT,#ifdef CONFIG_ZONE_DEVICE SECTION_TAINT_ZONE_DEVICE_BIT,#endif SECTION_MAP_LAST_BIT,};
#define SECTION_HAS_MEM_MAP BIT(SECTION_HAS_MEM_MAP_BIT)#define SECTION_IS_ONLINE BIT(SECTION_IS_ONLINE_BIT)#define SECTION_IS_EARLY BIT(SECTION_IS_EARLY_BIT)#ifdef CONFIG_ZONE_DEVICE#define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT)#endif#define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1))#define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT
static inline struct page *__section_mem_map_addr(struct mem_section *section){ unsigned long map = section->section_mem_map; map &= SECTION_MAP_MASK; return (struct page *)map;}
static inline int valid_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));}
static inline int early_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_IS_EARLY));}
static inline int valid_section_nr(unsigned long nr){ return valid_section(__nr_to_section(nr));}
static inline int early_section_nr(unsigned long nr){ return early_section(__nr_to_section(nr));}
static inline int online_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_IS_ONLINE));}
#ifdef CONFIG_ZONE_DEVICEstatic inline int online_device_section(const struct mem_section *section){ unsigned long flags = SECTION_IS_ONLINE | SECTION_TAINT_ZONE_DEVICE;
return section && ((section->section_mem_map & flags) == flags);}
static inline struct zone *device_zone(int nid){ return &NODE_DATA(nid)->node_zones[ZONE_DEVICE];}#elsestatic inline int online_device_section(const struct mem_section *section){ return 0;}
static inline struct zone *device_zone(int nid){ return NULL;}#endif
static inline int online_section_nr(unsigned long nr){ return online_section(__nr_to_section(nr));}
#ifdef CONFIG_MEMORY_HOTPLUGvoid online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);#endif
static inline struct mem_section *__pfn_to_section(unsigned long pfn){ return __nr_to_section(pfn_to_section_nr(pfn));}
extern unsigned long __highest_used_section_nr;
static inline int subsection_map_index(unsigned long pfn){ return (pfn & ~(PAGE_SECTION_MASK)) / PAGES_PER_SUBSECTION;}
#ifdef CONFIG_SPARSEMEM_VMEMMAPstatic inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn){ int idx = subsection_map_index(pfn); struct mem_section_usage *usage = READ_ONCE(ms->usage);
return usage ? test_bit(idx, usage->subsection_map) : 0;}
static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn){ struct mem_section_usage *usage = READ_ONCE(ms->usage); int idx = subsection_map_index(*pfn); unsigned long bit;
if (!usage) return false;
if (test_bit(idx, usage->subsection_map)) return true;
/* Find the next subsection that exists */ bit = find_next_bit(usage->subsection_map, SUBSECTIONS_PER_SECTION, idx); if (bit == SUBSECTIONS_PER_SECTION) return false;
*pfn = (*pfn & PAGE_SECTION_MASK) + (bit * PAGES_PER_SUBSECTION); return true;}#elsestatic inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn){ return 1;}
static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn){ return true;}#endif
#ifndef CONFIG_HAVE_ARCH_PFN_VALID/** * pfn_valid - check if there is a valid memory map entry for a PFN * @pfn: the page frame number to check * * Check if there is a valid memory map entry aka struct page for the @pfn. * Note, that availability of the memory map entry does not imply that * there is actual usable memory at that @pfn. The struct page may * represent a hole or an unusable page frame. * * Return: 1 for PFNs that have memory map entries and 0 otherwise */static inline int pfn_valid(unsigned long pfn){ struct mem_section *ms; int ret;
/* * Ensure the upper PAGE_SHIFT bits are clear in the * pfn. Else it might lead to false positives when * some of the upper bits are set, but the lower bits * match a valid pfn. */ if (PHYS_PFN(PFN_PHYS(pfn)) != pfn) return 0;
if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) return 0; ms = __pfn_to_section(pfn); rcu_read_lock_sched(); if (!valid_section(ms)) { rcu_read_unlock_sched(); return 0; } /* * Traditionally early sections always returned pfn_valid() for * the entire section-sized span. */ ret = early_section(ms) || pfn_section_valid(ms, pfn); rcu_read_unlock_sched();
return ret;}
/* Returns end_pfn or higher if no valid PFN remaining in range */static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end_pfn){ unsigned long nr = pfn_to_section_nr(pfn);
rcu_read_lock_sched();
while (nr <= __highest_used_section_nr && pfn < end_pfn) { struct mem_section *ms = __pfn_to_section(pfn);
if (valid_section(ms) && (early_section(ms) || pfn_section_first_valid(ms, &pfn))) { rcu_read_unlock_sched(); return pfn; }
/* Nothing left in this section? Skip to next section */ nr++; pfn = section_nr_to_pfn(nr); }
rcu_read_unlock_sched(); return end_pfn;}
static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_pfn){ pfn++;
if (pfn >= end_pfn) return end_pfn;
/* * Either every PFN within the section (or subsection for VMEMMAP) is * valid, or none of them are. So there's no point repeating the check * for every PFN; only call first_valid_pfn() again when crossing a * (sub)section boundary (i.e. !(pfn & ~PAGE_{SUB,}SECTION_MASK)). */ if (pfn & ~(IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP) ? PAGE_SUBSECTION_MASK : PAGE_SECTION_MASK)) return pfn;
return first_valid_pfn(pfn, end_pfn);}
#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn) \ for ((_pfn) = first_valid_pfn((_start_pfn), (_end_pfn)); \ (_pfn) < (_end_pfn); \ (_pfn) = next_valid_pfn((_pfn), (_end_pfn)))
#endif
static inline unsigned long next_early_section_nr(unsigned long section_nr){ while (++section_nr <= __highest_used_section_nr) { if (early_section_nr(section_nr)) return section_nr; }
return -1;}
#define for_each_early_section_nr(start, section_nr) \ for (section_nr = next_early_section_nr(start - 1); \ section_nr != -1; \ section_nr = next_early_section_nr(section_nr))
/* * These are _only_ used during initialisation, therefore they * can use __initdata ... They could have names to indicate * this restriction. */#ifdef CONFIG_NUMA#define pfn_to_nid(pfn) \({ \ unsigned long __pfn_to_nid_pfn = (pfn); \ page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \})#else#define pfn_to_nid(pfn) (0)#endif#endif /* CONFIG_SPARSEMEM */
/* * Fallback case for when the architecture provides its own pfn_valid() but * not a corresponding for_each_valid_pfn(). */#ifndef for_each_valid_pfn#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn) \ for ((_pfn) = (_start_pfn); (_pfn) < (_end_pfn); (_pfn)++) \ if (pfn_valid(_pfn))#endif
#endif /* !__GENERATING_BOUNDS.H */#endif /* !__ASSEMBLER__ */#endif /* _LINUX_MMZONE_H */