Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033/* SPDX-License-Identifier: GPL-2.0-or-later *//* memcontrol.h - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> */
#ifndef _LINUX_MEMCONTROL_H#define _LINUX_MEMCONTROL_H#include <linux/cgroup.h>#include <linux/vm_event_item.h>#include <linux/hardirq.h>#include <linux/jump_label.h>#include <linux/kernel.h>#include <linux/page_counter.h>#include <linux/vmpressure.h>#include <linux/eventfd.h>#include <linux/mm.h>#include <linux/vmstat.h>#include <linux/writeback.h>#include <linux/page-flags.h>#include <linux/shrinker.h>#include <linux/irq_work_types.h>
struct mem_cgroup;struct obj_cgroup;struct page;struct mm_struct;struct kmem_cache;struct swap_cluster_info;
/* Cgroup-specific page state, on top of universal node page state */enum memcg_stat_item { MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS, MEMCG_SOCK, MEMCG_PERCPU_B, MEMCG_KMEM, MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, MEMCG_NR_STAT,};
enum memcg_memory_event { MEMCG_LOW, MEMCG_HIGH, MEMCG_MAX, MEMCG_OOM, MEMCG_OOM_KILL, MEMCG_OOM_GROUP_KILL, MEMCG_SWAP_HIGH, MEMCG_SWAP_MAX, MEMCG_SWAP_FAIL, MEMCG_SOCK_THROTTLED, MEMCG_NR_MEMORY_EVENTS,};
struct mem_cgroup_reclaim_cookie { pg_data_t *pgdat; int generation;};
#ifdef CONFIG_MEMCG
#define MEM_CGROUP_ID_SHIFT 16
struct memcg_vmstats_percpu;struct memcg1_events_percpu;struct memcg_vmstats;struct lruvec_stats_percpu;struct lruvec_stats;
struct mem_cgroup_reclaim_iter { struct mem_cgroup *position; /* scan generation, increased every round-trip */ atomic_t generation;};
/* * per-node information in memory controller. */struct mem_cgroup_per_node { /* Set when the memcg is created, then only read. */ __cacheline_group_begin_aligned(memcg_pn_read_mostly); struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */
struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; struct lruvec_stats *lruvec_stats; struct shrinker_info __rcu *shrinker_info; struct obj_cgroup __rcu *objcg;
__cacheline_group_end_aligned(memcg_pn_read_mostly);
/* * Keep lruvec on its own lines. Sharing them with lru_zone_size[] * regressed, see commit f59adcf59332 ("mm: memcg: add cacheline * padding after lruvec in mem_cgroup_per_node"). */ __cacheline_group_begin_aligned(memcg_pn_lruvec); struct lruvec lruvec; __cacheline_group_end_aligned(memcg_pn_lruvec);
/* Written on every LRU update and on every reclaim iteration. */ __cacheline_group_begin_aligned(memcg_pn_write_hot); long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter;#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* slab stats for nmi context */ atomic_t slab_reclaimable; atomic_t slab_unreclaimable;#endif __cacheline_group_end_aligned(memcg_pn_write_hot);
/* Touched only when the memcg is reparented or freed. */ __cacheline_group_begin_aligned(memcg_pn_cold); /* * orig_objcg preserves a pointer (and a reference) to the original * objcg until the end of life of memcg. */ struct obj_cgroup *orig_objcg; /* list of inherited objcgs, protected by objcg_lock */ struct list_head objcg_list; __cacheline_group_end_aligned(memcg_pn_cold);};
struct mem_cgroup_threshold { struct eventfd_ctx *eventfd; unsigned long threshold;};
/* For threshold */struct mem_cgroup_threshold_ary { /* An array index points to threshold just below or equal to usage. */ int current_threshold; /* Size of entries[] */ unsigned int size; /* Array of thresholds */ struct mem_cgroup_threshold entries[] __counted_by(size);};
struct mem_cgroup_thresholds { /* Primary thresholds array */ struct mem_cgroup_threshold_ary *primary; /* * Spare threshold array. * This is needed to make mem_cgroup_unregister_event() "never fail". * It must be able to store at least primary->size - 1 entries. */ struct mem_cgroup_threshold_ary *spare;};
/* * Remember four most recent foreign writebacks with dirty pages in this * cgroup. Inode sharing is expected to be uncommon and, even if we miss * one in a given round, we're likely to catch it later if it keeps * foreign-dirtying, so a fairly low count should be enough. * * See mem_cgroup_track_foreign_dirty_slowpath() for details. */#define MEMCG_CGWB_FRN_CNT 4
struct memcg_cgwb_frn { u64 bdi_id; /* bdi->id of the foreign inode */ int memcg_id; /* memcg->css.id of foreign inode */ u64 at; /* jiffies_64 at the time of dirtying */ struct wb_completion done; /* tracks in-flight foreign writebacks */};
/* * Bucket for arbitrarily byte-sized objects charged to a memory * cgroup. The bucket can be reparented in one piece when the cgroup * is destroyed, without having to round up the individual references * of all live memory objects in the wild. */struct obj_cgroup { struct percpu_ref refcnt; struct mem_cgroup *memcg; atomic_t nr_charged_bytes; refcount_t private_id_ref; union { struct list_head list; /* protected by objcg_lock */ struct rcu_head rcu; }; bool is_root;};
/* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide * statistics based on the statistics developed by Rik Van Riel for clock-pro, * to help the administrator determine what knobs to tune. */struct mem_cgroup { struct cgroup_subsys_state css;
/* Accounted resources */ struct page_counter memory; /* Both v1 & v2 */
union { struct page_counter swap; /* v2 only */ struct page_counter memsw; /* v1 only */ };
/* Written on the charge, reclaim and socket paths. */ __cacheline_group_begin_aligned(memcg_write_hot); /* * Hint of reclaim pressure for socket memory management. Note * that this indicator should NOT be used in legacy cgroup mode * where socket memory is accounted/charged separately. */ u64 socket_pressure;#if BITS_PER_LONG < 64 seqlock_t socket_pressure_seqlock;#endif /* * memory.events is bumped for this memcg and all its ancestors, so a * busy child dirties every ancestor. */ atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS];
/* vmpressure notifications. Written on every reclaim iteration. */ struct vmpressure vmpressure;
#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* MEMCG_KMEM for nmi context */ atomic_t kmem_stat;#endif
/* Range enforcement for interrupt charges */ struct irq_work high_irq_work; struct work_struct high_work;
__cacheline_group_end_aligned(memcg_write_hot);
/* * Off the charge and fault paths. Not write free: cgwb_domain is * written on every writeout completion and mm_list on fork, exit and * MGLRU aging. They are grouped here so those writes cannot land on * a line that the fast paths read. */ __cacheline_group_begin_aligned(memcg_cold); /* registered local peak watchers */ struct list_head memory_peaks; struct list_head swap_peaks; spinlock_t peaks_lock;
/* memory.events and memory.events.local */ struct cgroup_file events_file; struct cgroup_file events_local_file;
/* handle for "memory.swap.events" */ struct cgroup_file swap_events_file;
#ifdef CONFIG_CGROUP_WRITEBACK struct list_head cgwb_list; struct wb_domain cgwb_domain; struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];#endif
#ifdef CONFIG_LRU_GEN_WALKS_MMU /* per-memcg mm_struct list */ struct lru_gen_mm_list mm_list;#endif __cacheline_group_end_aligned(memcg_cold);
#ifdef CONFIG_MEMCG_V1 /* v1 only. Not grouped: v1 is legacy, sorting it is not worth it. */
/* Legacy consumer-oriented counters */ struct page_counter kmem; /* v1 only */ struct page_counter tcpmem; /* v1 only */
struct memcg1_events_percpu __percpu *events_percpu;
/* protected by memcg_oom_lock */ bool oom_lock; int under_oom;
/* OOM-Killer disable */ int oom_kill_disable;
/* protect arrays of thresholds */ struct mutex thresholds_lock;
/* thresholds for memory usage. RCU-protected */ struct mem_cgroup_thresholds thresholds;
/* thresholds for mem+swap usage. RCU-protected */ struct mem_cgroup_thresholds memsw_thresholds;
/* For oom notifier event fd */ struct list_head oom_notify;
/* Legacy tcp memory accounting */ bool tcpmem_active; int tcpmem_pressure;
/* List of events which userspace want to receive */ struct list_head event_list; spinlock_t event_list_lock;
int swappiness;#endif /* CONFIG_MEMCG_V1 */
/* * Set when the memcg is created and cleared when it is offlined. * Never written on a hot path. */ __cacheline_group_begin_aligned(memcg_read_mostly); /* Read on every stat update */ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
/* memory.stat */ struct memcg_vmstats *vmstats;
#ifdef CONFIG_ZSWAP unsigned long zswap_max;#endif
/* The objcg holding private memcg ID. */ struct obj_cgroup *private_id_objcg;
/* Private memcg ID. Used to ID objects that outlive the cgroup */ int private_id;
int kmemcg_id;
/* * Should the OOM killer kill all belonging tasks, had it kill one? */ bool oom_group;
#ifdef CONFIG_ZSWAP /* * Prevent pages from this memcg from being written back from zswap to * swap, and from being swapped out on zswap store failures. */ bool zswap_writeback;#endif /* Not padded: nodeinfo[] is read-mostly too, let it share the line. */ __cacheline_group_end(memcg_read_mostly);
struct mem_cgroup_per_node *nodeinfo[];};
/* * size of first charge trial. * TODO: maybe necessary to use big numbers in big irons or dynamic based of the * workload. */#define MEMCG_CHARGE_BATCH 64U
extern struct mem_cgroup *root_mem_cgroup;
enum page_memcg_data_flags { /* page->memcg_data is a pointer to an slabobj_ext vector */ MEMCG_DATA_OBJEXTS = (1UL << 0), /* page has been accounted as a non-slab kernel page */ MEMCG_DATA_KMEM = (1UL << 1), /* the next bit after the last actual flag */ __NR_MEMCG_DATA_FLAGS = (1UL << 2),};
#define __OBJEXTS_ALLOC_FAIL MEMCG_DATA_OBJEXTS#define __FIRST_OBJEXT_FLAG __NR_MEMCG_DATA_FLAGS
#else /* CONFIG_MEMCG */
#define __OBJEXTS_ALLOC_FAIL (1UL << 0)#define __FIRST_OBJEXT_FLAG (1UL << 0)
#endif /* CONFIG_MEMCG */
enum objext_flags { /* * Use bit 0 with zero other bits to signal that slabobj_ext vector * failed to allocate. The same bit 0 with valid upper bits means * MEMCG_DATA_OBJEXTS. */ OBJEXTS_ALLOC_FAIL = __OBJEXTS_ALLOC_FAIL, __OBJEXTS_FLAG_UNUSED = __FIRST_OBJEXT_FLAG, /* the next bit after the last actual flag */ __NR_OBJEXTS_FLAGS = (__FIRST_OBJEXT_FLAG << 1),};
#define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1)
#ifdef CONFIG_MEMCG/* * After the initialization objcg->memcg is always pointing at * a valid memcg, but can be atomically swapped to the parent memcg. * * The caller must ensure that the returned memcg won't be released. */static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg){ lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex)); return objcg ? READ_ONCE(objcg->memcg) : NULL;}
/* * folio_objcg - get the object cgroup associated with a folio. * @folio: Pointer to the folio. * * Returns a pointer to the object cgroup associated with the folio, * or NULL. This function assumes that the folio is known to have a * proper object cgroup pointer. */static inline struct obj_cgroup *folio_objcg(const struct folio *folio){ unsigned long memcg_data = folio->memcg_data;
VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);}
/* * folio_memcg - Get the memory cgroup associated with a folio. * @folio: Pointer to the folio. * * Returns a pointer to the memory cgroup associated with the folio, * or NULL. This function assumes that the folio is known to have a * proper memory cgroup pointer. It's not safe to call this function * against some type of folios, e.g. slab folios or ex-slab folios. * * For a folio any of the following ensures folio and objcg binding stability: * * - the folio lock * - LRU isolation * - exclusive reference * * Based on the stable binding of folio and objcg, for a folio any of the * following ensures folio and memcg binding stability: * * - cgroup_mutex * - the lruvec lock * * If the caller only want to ensure that the page counters of memcg are * updated correctly, ensure that the binding stability of folio and objcg * is sufficient. * * Note: The caller should hold an rcu read lock or cgroup_mutex to protect * memcg associated with a folio from being released. */static inline struct mem_cgroup *folio_memcg(const struct folio *folio){ struct obj_cgroup *objcg = folio_objcg(folio);
return obj_cgroup_memcg(objcg);}
/* * folio_memcg_charged - If a folio is charged to a memory cgroup. * @folio: Pointer to the folio. * * Returns true if folio is charged to a memory cgroup, otherwise returns false. */static inline bool folio_memcg_charged(const struct folio *folio){ return folio->memcg_data != 0;}
/* * folio_memcg_check - Get the memory cgroup associated with a folio. * @folio: Pointer to the folio. * * Returns a pointer to the memory cgroup associated with the folio, * or NULL. This function unlike folio_memcg() can take any folio * as an argument. It has to be used in cases when it's not known if a folio * has an associated memory cgroup pointer or an object cgroups vector or * an object cgroup. * * The page and objcg or memcg binding rules can refer to folio_memcg(). * * A caller should hold an rcu read lock to protect memcg associated with a * page from being released. */static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio){ /* * Because folio->memcg_data might be changed asynchronously * for slabs, READ_ONCE() should be used here. */ unsigned long memcg_data = READ_ONCE(folio->memcg_data); struct obj_cgroup *objcg;
if (memcg_data & MEMCG_DATA_OBJEXTS) return NULL;
objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
return obj_cgroup_memcg(objcg);}
static inline struct mem_cgroup *page_memcg_check(const struct page *page){ if (PageTail(page)) return NULL; return folio_memcg_check((const struct folio *)page);}
static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg){ struct mem_cgroup *memcg;
rcu_read_lock();retry: memcg = obj_cgroup_memcg(objcg); if (unlikely(!css_tryget(&memcg->css))) goto retry; rcu_read_unlock();
return memcg;}
/* * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set. * @folio: Pointer to the folio. * * Checks if the folio has MemcgKmem flag set. The caller must ensure * that the folio has an associated memory cgroup. It's not safe to call * this function against some types of folios, e.g. slab folios. */static inline bool folio_memcg_kmem(const struct folio *folio){ VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio); return folio->memcg_data & MEMCG_DATA_KMEM;}
static inline bool PageMemcgKmem(const struct page *page){ return folio_memcg_kmem(page_folio(page));}
static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg){ return (memcg == root_mem_cgroup);}
/** * mem_cgroup_shrink_is_root - is this a global or root-memcg shrink invocation? * @sc: shrink_control describing the current shrinker call * * Returns true when @sc represents a global reclaim shrink (sc->memcg == NULL) * or a root-memcg shrink, i.e. not a per-memcg iteration of * shrink_slab_memcg(). Filesystems whose ->nr_cached_objects()/ * ->free_cached_objects() implementations operate on filesystem-global state * and do not honour sc->memcg can use this to early-return 0 in per-memcg * contexts. */static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc){ return !sc->memcg || mem_cgroup_is_root(sc->memcg);}
static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg){ return objcg->is_root;}
static inline bool mem_cgroup_disabled(void){ return !cgroup_subsys_enabled(memory_cgrp_subsys);}
static inline void mem_cgroup_protection(const struct mem_cgroup *root, const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage){ *min = *low = *usage = 0;
if (mem_cgroup_disabled()) return;
*usage = page_counter_read(&memcg->memory); /* * There is no reclaim protection applied to a targeted reclaim. * We are special casing this specific case here because * mem_cgroup_calculate_protection is not robust enough to keep * the protection invariant for calculated effective values for * parallel reclaimers with different reclaim target. This is * especially a problem for tail memcgs (as they have pages on LRU) * which would want to have effective values 0 for targeted reclaim * but a different value for external reclaim. * * Example * Let's have global and A's reclaim in parallel: * | * A (low=2G, usage = 3G, max = 3G, children_low_usage = 1.5G) * |\ * | C (low = 1G, usage = 2.5G) * B (low = 1G, usage = 0.5G) * * For the global reclaim * A.elow = A.low * B.elow = min(B.usage, B.low) because children_low_usage <= A.elow * C.elow = min(C.usage, C.low) * * With the effective values resetting we have A reclaim * A.elow = 0 * B.elow = B.low * C.elow = C.low * * If the global reclaim races with A's reclaim then * B.elow = C.elow = 0 because children_low_usage > A.elow) * is possible and reclaiming B would be violating the protection. * */ if (root == memcg) return;
*min = READ_ONCE(memcg->memory.emin); *low = READ_ONCE(memcg->memory.elow);}
void mem_cgroup_calculate_protection(struct mem_cgroup *root, struct mem_cgroup *memcg);
static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, const struct mem_cgroup *memcg){ /* * The root memcg doesn't account charges, and doesn't support * protection. The target memcg's protection is ignored, see * mem_cgroup_calculate_protection() and mem_cgroup_protection() */ return mem_cgroup_disabled() || mem_cgroup_is_root(memcg) || memcg == target;}
static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, const struct mem_cgroup *memcg){ if (mem_cgroup_unprotected(target, memcg)) return false;
return READ_ONCE(memcg->memory.elow) >= page_counter_read(&memcg->memory);}
static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, const struct mem_cgroup *memcg){ if (mem_cgroup_unprotected(target, memcg)) return false;
return READ_ONCE(memcg->memory.emin) >= page_counter_read(&memcg->memory);}
int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
/** * mem_cgroup_charge - Charge a newly allocated folio to a cgroup. * @folio: Folio to charge. * @mm: mm context of the allocating task. * @gfp: Reclaim mode. * * Try to charge @folio to the memcg that @mm belongs to, reclaiming * pages according to @gfp if necessary. If @mm is NULL, try to * charge to the active memcg. * * Do not use this for folios allocated for swapin. * * Return: 0 on success. Otherwise, an error code is returned. */static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp){ if (mem_cgroup_disabled()) return 0; return __mem_cgroup_charge(folio, mm, gfp);}
int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp);
void __mem_cgroup_uncharge(struct folio *folio);
/** * mem_cgroup_uncharge - Uncharge a folio. * @folio: Folio to uncharge. * * Uncharge a folio previously charged with mem_cgroup_charge(). */static inline void mem_cgroup_uncharge(struct folio *folio){ if (mem_cgroup_disabled()) return; __mem_cgroup_uncharge(folio);}
void __mem_cgroup_uncharge_folios(struct folio_batch *folios);static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios){ if (mem_cgroup_disabled()) return; __mem_cgroup_uncharge_folios(folios);}
void mem_cgroup_replace_folio(struct folio *old, struct folio *new);void mem_cgroup_migrate(struct folio *old, struct folio *new);
/** * mem_cgroup_lruvec - get the lru list vector for a memcg & node * @memcg: memcg of the wanted lruvec * @pgdat: pglist_data * * Returns the lru list vector holding pages for a given @memcg & * @pgdat combination. This can be the node lruvec, if the memory * controller is disabled. */static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat){ struct mem_cgroup_per_node *mz; struct lruvec *lruvec;
if (mem_cgroup_disabled()) { lruvec = &pgdat->__lruvec; goto out; }
if (!memcg) memcg = root_mem_cgroup;
mz = memcg->nodeinfo[pgdat->node_id]; lruvec = &mz->lruvec;out: /* * Since a node can be onlined after the mem_cgroup was created, * we have to be prepared to initialize lruvec->pgdat here; * and if offlined then reonlined, we need to reinitialize it. */ if (unlikely(lruvec->pgdat != pgdat)) lruvec->pgdat = pgdat; return lruvec;}
/** * folio_lruvec - return lruvec for isolating/putting an LRU folio * @folio: Pointer to the folio. * * Call with rcu_read_lock() held to ensure the lifetime of the returned lruvec. * Note that this alone will NOT guarantee the stability of the folio->lruvec * association; the folio can be reparented to an ancestor if this races with * cgroup deletion. * * Use folio_lruvec_lock() to ensure both lifetime and stability of the binding. * Once a lruvec is locked, folio_lruvec() can be called on other folios, and * their binding is stable if the returned lruvec matches the one the caller has * locked. Useful for lock batching. */static inline struct lruvec *folio_lruvec(const struct folio *folio){ struct mem_cgroup *memcg = folio_memcg(folio);
VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio); return mem_cgroup_lruvec(memcg, folio_pgdat(folio));}
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
struct mem_cgroup *get_mem_cgroup_from_current(void);
struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio);
struct lruvec *folio_lruvec_lock(const struct folio *folio);struct lruvec *folio_lruvec_lock_irq(const struct folio *folio);struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags);
static inlinestruct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ return css ? container_of(css, struct mem_cgroup, css) : NULL;}
static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg){ if (obj_cgroup_is_root(objcg)) return true; return percpu_ref_tryget(&objcg->refcnt);}
static inline void obj_cgroup_get_many(struct obj_cgroup *objcg, unsigned long nr){ if (!obj_cgroup_is_root(objcg)) percpu_ref_get_many(&objcg->refcnt, nr);}
static inline void obj_cgroup_get(struct obj_cgroup *objcg){ obj_cgroup_get_many(objcg, 1);}
static inline void obj_cgroup_put(struct obj_cgroup *objcg){ if (objcg && !obj_cgroup_is_root(objcg)) percpu_ref_put(&objcg->refcnt);}
static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg){ return !memcg || css_tryget(&memcg->css);}
static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg){ return !memcg || css_tryget_online(&memcg->css);}
static inline void mem_cgroup_put(struct mem_cgroup *memcg){ if (memcg) css_put(&memcg->css);}
#define mem_cgroup_from_counter(counter, member) \ container_of(counter, struct mem_cgroup, member)
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, struct mem_cgroup *, struct mem_cgroup_reclaim_cookie *);void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*)(struct task_struct *, void *), void *arg);
static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg){ if (mem_cgroup_disabled()) return 0;
return memcg->private_id;}struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id);
static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg){ return memcg ? cgroup_id(memcg->css.cgroup) : 0;}
struct mem_cgroup *mem_cgroup_get_from_id(u64 id);
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m){ return mem_cgroup_from_css(seq_css(m));}
static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec){ const struct mem_cgroup_per_node *mz;
if (mem_cgroup_disabled()) return NULL;
mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return mz->memcg;}
/** * parent_mem_cgroup - find the accounting parent of a memcg * @memcg: memcg whose parent to find * * Returns the parent memcg, or NULL if this is the root. */static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg){ return mem_cgroup_from_css(memcg->css.parent);}
static inline bool mem_cgroup_is_descendant(const struct mem_cgroup *memcg, const struct mem_cgroup *root){ if (root == memcg) return true; return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);}
static inline bool mm_match_cgroup(struct mm_struct *mm, const struct mem_cgroup *memcg){ struct mem_cgroup *task_memcg; bool match = false;
rcu_read_lock(); task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (task_memcg) match = mem_cgroup_is_descendant(task_memcg, memcg); rcu_read_unlock(); return match;}
struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio);ino_t page_cgroup_ino(const struct page *page);
static inline bool mem_cgroup_online(struct mem_cgroup *memcg){ if (mem_cgroup_disabled()) return true; return css_is_online(&memcg->css);}
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages);
static inlineunsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx){ long val; const struct mem_cgroup_per_node *mz;
mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); if (WARN_ON_ONCE(val < 0)) return 0;
return val;}
void __mem_cgroup_handle_over_high(gfp_t gfp_mask);
static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask){ if (unlikely(current->memcg_nr_pages_over_high)) __mem_cgroup_handle_over_high(gfp_mask);}
unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg);
void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p);
void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, struct mem_cgroup *oom_domain);void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
/* idx can be of type enum memcg_stat_item or node_stat_item */void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, int val);
static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return;
rcu_read_lock(); memcg = folio_memcg(page_folio(page)); if (memcg) mod_memcg_state(memcg, idx, val); rcu_read_unlock();}
unsigned long memcg_events(const struct mem_cgroup *memcg, int event);unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx);unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item);bool memcg_stat_item_valid(int idx);bool memcg_vm_event_item_valid(enum vm_event_item idx);unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx);unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx);unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx);
void mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val);
void mem_cgroup_flush_stats(struct mem_cgroup *memcg);void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg);
void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val);
void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count);
static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr){ struct mem_cgroup *memcg;
if (!folio_memcg_charged(folio)) return;
rcu_read_lock(); memcg = folio_memcg(folio); count_memcg_events(memcg, idx, nr); rcu_read_unlock();}
static inline void count_memcg_events_mm(struct mm_struct *mm, enum vm_event_item idx, unsigned long count){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return;
rcu_read_lock(); memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (likely(memcg)) count_memcg_events(memcg, idx, count); rcu_read_unlock();}
static inline void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx){ count_memcg_events_mm(mm, idx, 1);}
void __memcg_memory_event(struct mem_cgroup *memcg, enum memcg_memory_event event, bool allow_spinning);
static inline void memcg_memory_event(struct mem_cgroup *memcg, enum memcg_memory_event event){ __memcg_memory_event(memcg, event, true);}
static inline void memcg_memory_event_mm(struct mm_struct *mm, enum memcg_memory_event event){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return;
rcu_read_lock(); memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (likely(memcg)) memcg_memory_event(memcg, event); rcu_read_unlock();}
void split_page_memcg(struct page *first, unsigned order);void folio_split_memcg_refs(struct folio *folio, unsigned old_order, unsigned new_order);
static inline u64 cgroup_id_from_mm(struct mm_struct *mm){ struct mem_cgroup *memcg; u64 id;
if (mem_cgroup_disabled()) return 0;
rcu_read_lock(); memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (!memcg) memcg = root_mem_cgroup; id = cgroup_id(memcg->css.cgroup); rcu_read_unlock(); return id;}
void mem_cgroup_flush_workqueue(void);
extern int mem_cgroup_init(void);#else /* CONFIG_MEMCG */
static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg){ return NULL;}
#define MEM_CGROUP_ID_SHIFT 0
#define root_mem_cgroup (NULL)
static inline struct mem_cgroup *folio_memcg(const struct folio *folio){ return NULL;}
static inline bool folio_memcg_charged(const struct folio *folio){ return false;}
static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio){ return NULL;}
static inline struct mem_cgroup *page_memcg_check(const struct page *page){ return NULL;}
static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg){ return NULL;}
static inline bool folio_memcg_kmem(const struct folio *folio){ return false;}
static inline bool PageMemcgKmem(const struct page *page){ return false;}
static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg){ return true;}
static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc){ return true;}
static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg){ return true;}
static inline bool mem_cgroup_disabled(void){ return true;}
static inline void memcg_memory_event(struct mem_cgroup *memcg, enum memcg_memory_event event){}
static inline void memcg_memory_event_mm(struct mm_struct *mm, enum memcg_memory_event event){}
static inline void mem_cgroup_protection(const struct mem_cgroup *root, const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage){ *min = *low = *usage = 0;}
static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root, struct mem_cgroup *memcg){}
static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, const struct mem_cgroup *memcg){ return true;}static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, const struct mem_cgroup *memcg){ return false;}
static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, const struct mem_cgroup *memcg){ return false;}
static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp){ return 0;}
static inline int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, gfp_t gfp){ return 0;}
static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp){ return 0;}
static inline void mem_cgroup_uncharge(struct folio *folio){}
static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios){}
static inline void mem_cgroup_replace_folio(struct folio *old, struct folio *new){}
static inline void mem_cgroup_migrate(struct folio *old, struct folio *new){}
static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat){ return &pgdat->__lruvec;}
static inline struct lruvec *folio_lruvec(const struct folio *folio){ struct pglist_data *pgdat = folio_pgdat(folio); return &pgdat->__lruvec;}
static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg){ return NULL;}
static inline bool mm_match_cgroup(struct mm_struct *mm, const struct mem_cgroup *memcg){ return true;}
static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm){ return NULL;}
static inline struct mem_cgroup *get_mem_cgroup_from_current(void){ return NULL;}
static inline struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio){ return NULL;}
static inlinestruct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ return NULL;}
static inline void obj_cgroup_get(struct obj_cgroup *objcg){}
static inline void obj_cgroup_put(struct obj_cgroup *objcg){}
static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg){ return true;}
static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg){ return true;}
static inline void mem_cgroup_put(struct mem_cgroup *memcg){}
static inline struct lruvec *folio_lruvec_lock(const struct folio *folio){ struct pglist_data *pgdat = folio_pgdat(folio);
rcu_read_lock(); spin_lock(&pgdat->__lruvec.lru_lock); return &pgdat->__lruvec;}
static inline struct lruvec *folio_lruvec_lock_irq(const struct folio *folio){ struct pglist_data *pgdat = folio_pgdat(folio);
rcu_read_lock(); spin_lock_irq(&pgdat->__lruvec.lru_lock); return &pgdat->__lruvec;}
static inline struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flagsp){ struct pglist_data *pgdat = folio_pgdat(folio);
rcu_read_lock(); spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp); return &pgdat->__lruvec;}
static inline struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, struct mem_cgroup *prev, struct mem_cgroup_reclaim_cookie *reclaim){ return NULL;}
static inline void mem_cgroup_iter_break(struct mem_cgroup *root, struct mem_cgroup *prev){}
static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*fn)(struct task_struct *, void *), void *arg){}
static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg){ return 0;}
static inline struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id){ WARN_ON_ONCE(id); /* XXX: This should always return root_mem_cgroup */ return NULL;}
static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg){ return 0;}
static inline struct mem_cgroup *mem_cgroup_get_from_id(u64 id){ return NULL;}
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m){ return NULL;}
static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec){ return NULL;}
static inline bool mem_cgroup_online(struct mem_cgroup *memcg){ return true;}
static inlineunsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx){ return 0;}
static inline unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg){ return 0;}
static inline voidmem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p){}
static inline voidmem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg){}
static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask){}
static inline struct mem_cgroup *mem_cgroup_get_oom_group( struct task_struct *victim, struct mem_cgroup *oom_domain){ return NULL;}
static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg){}
static inline void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, int nr){}
static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val){}
static inline unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx){ return 0;}
static inline unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item){ return 0;}
static inline bool memcg_stat_item_valid(int idx){ return false;}
static inline bool memcg_vm_event_item_valid(enum vm_event_item idx){ return false;}
static inline unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx){ return node_page_state(lruvec_pgdat(lruvec), idx);}
static inline unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx){ return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);}
static inline unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx){ return node_page_state(lruvec_pgdat(lruvec), idx);}
static inline void mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val){}
static inline void mem_cgroup_flush_stats(struct mem_cgroup *memcg){}
static inline void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg){}
static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val){ struct page *page = virt_to_head_page(p);
mod_node_page_state(page_pgdat(page), idx, val);}
static inline void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count){}
static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr){}
static inline void count_memcg_events_mm(struct mm_struct *mm, enum vm_event_item idx, unsigned long count){}
static inlinevoid count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx){}
static inline void split_page_memcg(struct page *first, unsigned order){}
static inline void folio_split_memcg_refs(struct folio *folio, unsigned old_order, unsigned new_order){}
static inline u64 cgroup_id_from_mm(struct mm_struct *mm){ return 0;}
static inline void mem_cgroup_flush_workqueue(void) { }
static inline int mem_cgroup_init(void) { return 0; }#endif /* CONFIG_MEMCG */
static inline struct lruvec *parent_lruvec(const struct lruvec *lruvec){ struct mem_cgroup *memcg;
memcg = lruvec_memcg(lruvec); if (!memcg) return NULL; memcg = parent_mem_cgroup(memcg); if (!memcg) return NULL; return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));}
static inline void lruvec_lock_irq(struct lruvec *lruvec){ rcu_read_lock(); spin_lock_irq(&lruvec->lru_lock);}
static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec){#ifdef CONFIG_MEMCG struct pglist_data *pgdat = lruvec_pgdat(lruvec); struct mem_cgroup *memcg = lruvec_memcg(lruvec);
rcu_read_lock();
/* * The memcg can be NULL when the memory controller is disabled. * Otherwise, the caller keeps the memcg owning @lruvec alive. */ while (unlikely(memcg && css_is_dying(&memcg->css))) { memcg = parent_mem_cgroup(memcg); lruvec = mem_cgroup_lruvec(memcg, pgdat); }
spin_lock_irq(&lruvec->lru_lock);#else lruvec_lock_irq(lruvec);#endif
return lruvec;}
static inline void lruvec_unlock(struct lruvec *lruvec){ spin_unlock(&lruvec->lru_lock); rcu_read_unlock();}
static inline void lruvec_unlock_irq(struct lruvec *lruvec){ spin_unlock_irq(&lruvec->lru_lock); rcu_read_unlock();}
static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long flags){ spin_unlock_irqrestore(&lruvec->lru_lock, flags); rcu_read_unlock();}
/* Test requires a stable folio->memcg binding, see folio_memcg() */static inline bool folio_matches_lruvec(const struct folio *folio, const struct lruvec *lruvec){ return lruvec_pgdat(lruvec) == folio_pgdat(folio) && lruvec_memcg(lruvec) == folio_memcg(folio);}
/* Don't lock again iff page's lruvec locked */static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio, struct lruvec *locked_lruvec){ if (locked_lruvec) { if (folio_matches_lruvec(folio, locked_lruvec)) return locked_lruvec;
lruvec_unlock_irq(locked_lruvec); }
return folio_lruvec_lock_irq(folio);}
/* Don't lock again iff folio's lruvec locked */static inline void folio_lruvec_relock_irqsave(const struct folio *folio, struct lruvec **lruvecp, unsigned long *flags){ if (*lruvecp) { if (folio_matches_lruvec(folio, *lruvecp)) return;
lruvec_unlock_irqrestore(*lruvecp, *flags); }
*lruvecp = folio_lruvec_lock_irqsave(folio, flags);}
#ifdef CONFIG_CGROUP_WRITEBACK
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, unsigned long *pheadroom, unsigned long *pdirty, unsigned long *pwriteback);
void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, struct bdi_writeback *wb);
static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, struct bdi_writeback *wb){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return;
if (!folio_memcg_charged(folio)) return;
rcu_read_lock(); memcg = folio_memcg(folio); if (unlikely(&memcg->css != wb->memcg_css)) mem_cgroup_track_foreign_dirty_slowpath(folio, wb); rcu_read_unlock();}
void mem_cgroup_flush_foreign(struct bdi_writeback *wb);
#else /* CONFIG_CGROUP_WRITEBACK */
static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb){ return NULL;}
static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, unsigned long *pheadroom, unsigned long *pdirty, unsigned long *pwriteback){}
static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, struct bdi_writeback *wb){}
static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb){}
#endif /* CONFIG_CGROUP_WRITEBACK */
struct sock;#ifdef CONFIG_MEMCGextern struct static_key_false memcg_sockets_enabled_key;#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
void mem_cgroup_sk_alloc(struct sock *sk);void mem_cgroup_sk_free(struct sock *sk);void mem_cgroup_sk_inherit(const struct sock *sk, struct sock *newsk);bool mem_cgroup_sk_charge(const struct sock *sk, unsigned int nr_pages, gfp_t gfp_mask);void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages);
#if BITS_PER_LONG < 64static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg){ u64 val = get_jiffies_64() + HZ; unsigned long flags;
write_seqlock_irqsave(&memcg->socket_pressure_seqlock, flags); memcg->socket_pressure = val; write_sequnlock_irqrestore(&memcg->socket_pressure_seqlock, flags);}
static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg){ unsigned int seq; u64 val;
do { seq = read_seqbegin(&memcg->socket_pressure_seqlock); val = memcg->socket_pressure; } while (read_seqretry(&memcg->socket_pressure_seqlock, seq));
return val;}#elsestatic inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg){ WRITE_ONCE(memcg->socket_pressure, jiffies + HZ);}
static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg){ return READ_ONCE(memcg->socket_pressure);}#endif
void shrinker_unit_cache_init(void);int alloc_shrinker_info(struct mem_cgroup *memcg);void free_shrinker_info(struct mem_cgroup *memcg);void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);void reparent_shrinker_deferred(struct mem_cgroup *memcg);
static inline int shrinker_id(struct shrinker *shrinker){ return shrinker->id;}#else#define mem_cgroup_sockets_enabled 0
static inline void mem_cgroup_sk_alloc(struct sock *sk){}
static inline void mem_cgroup_sk_free(struct sock *sk){}
static inline void mem_cgroup_sk_inherit(const struct sock *sk, struct sock *newsk){}
static inline bool mem_cgroup_sk_charge(const struct sock *sk, unsigned int nr_pages, gfp_t gfp_mask){ return false;}
static inline void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages){}
static inline void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id){}
static inline int shrinker_id(struct shrinker *shrinker){ return -1;}#endif
#ifdef CONFIG_MEMCGbool mem_cgroup_kmem_disabled(void);int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order);void __memcg_kmem_uncharge_page(struct page *page, int order);
/* * The returned objcg pointer is safe to use without additional * protection within a scope. The scope is defined either by * the current task (similar to the "current" global variable) * or by set_active_memcg() pair. * Please, use obj_cgroup_get() to get a reference if the pointer * needs to be used outside of the local scope. */struct obj_cgroup *current_obj_cgroup(void);struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio);
static inline struct obj_cgroup *get_obj_cgroup_from_current(void){ struct obj_cgroup *objcg = current_obj_cgroup();
if (objcg) obj_cgroup_get(objcg);
return objcg;}
int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size);void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size);
extern struct static_key_false memcg_bpf_enabled_key;static inline bool memcg_bpf_enabled(void){ return static_branch_likely(&memcg_bpf_enabled_key);}
extern struct static_key_false memcg_kmem_online_key;
static inline bool memcg_kmem_online(void){ return static_branch_likely(&memcg_kmem_online_key);}
static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order){ if (memcg_kmem_online()) return __memcg_kmem_charge_page(page, gfp, order); return 0;}
static inline void memcg_kmem_uncharge_page(struct page *page, int order){ if (memcg_kmem_online()) __memcg_kmem_uncharge_page(page, order);}
/* * A helper for accessing memcg's kmem_id, used for getting * corresponding LRU lists. */static inline int memcg_kmem_id(const struct mem_cgroup *memcg){ return memcg ? memcg->kmemcg_id : -1;}
struct mem_cgroup *mem_cgroup_from_virt(void *p);
static inline void count_objcg_events(struct obj_cgroup *objcg, enum vm_event_item idx, unsigned long count){ struct mem_cgroup *memcg;
if (!memcg_kmem_online()) return;
rcu_read_lock(); memcg = obj_cgroup_memcg(objcg); count_memcg_events(memcg, idx, count); rcu_read_unlock();}
void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask);
void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg);
static inline bool memcg_is_dying(struct mem_cgroup *memcg){ return memcg ? css_is_dying(&memcg->css) : false;}
#elsestatic inline bool mem_cgroup_kmem_disabled(void){ return true;}
static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order){ return 0;}
static inline void memcg_kmem_uncharge_page(struct page *page, int order){}
static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order){ return 0;}
static inline void __memcg_kmem_uncharge_page(struct page *page, int order){}
static inline struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio){ return NULL;}
static inline bool memcg_bpf_enabled(void){ return false;}
static inline bool memcg_kmem_online(void){ return false;}
static inline int memcg_kmem_id(const struct mem_cgroup *memcg){ return -1;}
static inline struct mem_cgroup *mem_cgroup_from_virt(void *p){ return NULL;}
static inline void count_objcg_events(struct obj_cgroup *objcg, enum vm_event_item idx, unsigned long count){}
static inline ino_t page_cgroup_ino(const struct page *page){ return 0;}
static inline void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask){}
static inline void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg){}
static inline bool memcg_is_dying(struct mem_cgroup *memcg){ return false;}#endif /* CONFIG_MEMCG */
#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN)void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, struct mem_cgroup *memcg);#elsestatic inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, struct mem_cgroup *memcg){}#endif
#if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP)bool obj_cgroup_may_zswap(struct obj_cgroup *objcg);void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size);void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size);bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg);#elsestatic inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg){ return true;}static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size){}static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size){}static inline bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg){ /* if zswap is disabled, do not block pages going to the swapping device */ return true;}#endif
/* Cgroup v1-related declarations */
#ifdef CONFIG_MEMCG_V1bool mem_cgroup_oom_synchronize(bool wait);
static inline bool task_in_memcg_oom(const struct task_struct *p){ return p->memcg_in_oom;}
static inline void mem_cgroup_enter_user_fault(void){ WARN_ON(current->in_user_fault); current->in_user_fault = 1;}
static inline void mem_cgroup_exit_user_fault(void){ WARN_ON(!current->in_user_fault); current->in_user_fault = 0;}
#else /* CONFIG_MEMCG_V1 */static inline bool task_in_memcg_oom(const struct task_struct *p){ return false;}
static inline bool mem_cgroup_oom_synchronize(bool wait){ return false;}
static inline void mem_cgroup_enter_user_fault(void){}
static inline void mem_cgroup_exit_user_fault(void){}
#endif /* CONFIG_MEMCG_V1 */
#if defined(CONFIG_MEMCG_V1) && defined(CONFIG_SWAP)
void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci);void memcg1_swapin(struct folio *folio);
#else
static inline void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci){}
static inline void memcg1_swapin(struct folio *folio){}#endif
#endif /* _LINUX_MEMCONTROL_H */