Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260// SPDX-License-Identifier: GPL-2.0/* * Memory subsystem support * * Written by Matt Tolentino <matthew.e.tolentino@intel.com> * Dave Hansen <haveblue@us.ibm.com> * * This file provides the necessary infrastructure to represent * a SPARSEMEM-memory-model system's physical memory in /sysfs. * All arch-independent code that assumes MEMORY_HOTPLUG requires * SPARSEMEM should be contained here, or in mm/memory_hotplug.c. */
#include <linux/module.h>#include <linux/init.h>#include <linux/topology.h>#include <linux/capability.h>#include <linux/device.h>#include <linux/memory.h>#include <linux/memory_hotplug.h>#include <linux/mm.h>#include <linux/stat.h>#include <linux/slab.h>#include <linux/xarray.h>#include <linux/export.h>
#include <linux/atomic.h>#include <linux/uaccess.h>
#define MEMORY_CLASS_NAME "memory"
static const char *const online_type_to_str[] = { [MMOP_OFFLINE] = "offline", [MMOP_ONLINE] = "online", [MMOP_ONLINE_KERNEL] = "online_kernel", [MMOP_ONLINE_MOVABLE] = "online_movable",};
int mhp_online_type_from_str(const char *str){ int i;
for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) { if (sysfs_streq(str, online_type_to_str[i])) return i; } return -EINVAL;}EXPORT_SYMBOL_GPL(mhp_online_type_from_str);
const char *mhp_online_type_to_str(int online_type){ if (online_type < 0 || online_type >= (int)ARRAY_SIZE(online_type_to_str)) return NULL; return online_type_to_str[online_type];}EXPORT_SYMBOL_GPL(mhp_online_type_to_str);
#define to_memory_block(dev) container_of(dev, struct memory_block, dev)
int sections_per_block;EXPORT_SYMBOL(sections_per_block);
static int memory_subsys_online(struct device *dev);static int memory_subsys_offline(struct device *dev);
static const struct bus_type memory_subsys = { .name = MEMORY_CLASS_NAME, .dev_name = MEMORY_CLASS_NAME, .online = memory_subsys_online, .offline = memory_subsys_offline,};
/* * Memory blocks are cached in a local radix tree to avoid * a costly linear search for the corresponding device on * the subsystem bus. */static DEFINE_XARRAY(memory_blocks);
/* * Memory groups, indexed by memory group id (mgid). */static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC);#define MEMORY_GROUP_MARK_DYNAMIC XA_MARK_1
static BLOCKING_NOTIFIER_HEAD(memory_chain);
int register_memory_notifier(struct notifier_block *nb){ return blocking_notifier_chain_register(&memory_chain, nb);}EXPORT_SYMBOL(register_memory_notifier);
void unregister_memory_notifier(struct notifier_block *nb){ blocking_notifier_chain_unregister(&memory_chain, nb);}EXPORT_SYMBOL(unregister_memory_notifier);
static void memory_block_release(struct device *dev){ struct memory_block *mem = to_memory_block(dev); /* Verify that the altmap is freed */ WARN_ON(mem->altmap); kfree(mem);}
/* Max block size to be set by memory_block_advise_max_size */static unsigned long memory_block_advised_size;static bool memory_block_advised_size_queried;
/** * memory_block_advise_max_size() - advise memory hotplug on the max suggested * block size, usually for alignment. * @size: suggestion for maximum block size. must be aligned on power of 2. * * Early boot software (pre-allocator init) may advise archs on the max block * size. This value can only decrease after initialization, as the intent is * to identify the largest supported alignment for all sources. * * Use of this value is arch-defined, as is min/max block size. * * Return: 0 on success * -EINVAL if size is 0 or not pow2 aligned * -EBUSY if value has already been probed */int __init memory_block_advise_max_size(unsigned long size){ if (!size || !is_power_of_2(size)) return -EINVAL;
if (memory_block_advised_size_queried) return -EBUSY;
if (memory_block_advised_size) memory_block_advised_size = min(memory_block_advised_size, size); else memory_block_advised_size = size;
return 0;}
/** * memory_block_advised_max_size() - query advised max hotplug block size. * * After the first call, the value can never change. Callers looking for the * actual block size should use memory_block_size_bytes. This interface is * intended for use by arch-init when initializing the hotplug block size. * * Return: advised size in bytes, or 0 if never set. */unsigned long memory_block_advised_max_size(void){ memory_block_advised_size_queried = true; return memory_block_advised_size;}
unsigned long __weak memory_block_size_bytes(void){ return MIN_MEMORY_BLOCK_SIZE;}EXPORT_SYMBOL_GPL(memory_block_size_bytes);
/* Show the memory block ID, relative to the memory block size */static ssize_t phys_index_show(struct device *dev, struct device_attribute *attr, char *buf){ struct memory_block *mem = to_memory_block(dev);
return sysfs_emit(buf, "%08lx\n", memory_block_id(mem->start_section_nr));}
/* * Legacy interface that we cannot remove. Always indicate "removable" * with CONFIG_MEMORY_HOTREMOVE - bad heuristic. */static ssize_t removable_show(struct device *dev, struct device_attribute *attr, char *buf){ return sysfs_emit(buf, "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE));}
/* * online, offline, going offline, etc. */static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf){ struct memory_block *mem = to_memory_block(dev); const char *output;
/* * We can probably put these states in a nice little array * so that they're not open-coded */ switch (mem->state) { case MEM_ONLINE: output = "online"; break; case MEM_OFFLINE: output = "offline"; break; case MEM_GOING_OFFLINE: output = "going-offline"; break; default: WARN_ON(1); return sysfs_emit(buf, "ERROR-UNKNOWN-%d\n", mem->state); }
return sysfs_emit(buf, "%s\n", output);}
int memory_notify(enum memory_block_state state, void *v){ return blocking_notifier_call_chain(&memory_chain, state, v);}
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)static unsigned long memblk_nr_poison(struct memory_block *mem);#elsestatic inline unsigned long memblk_nr_poison(struct memory_block *mem){ return 0;}#endif
/* * Must acquire mem_hotplug_lock in write mode. */static int memory_block_online(struct memory_block *mem){ unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; unsigned long nr_vmemmap_pages = 0; struct zone *zone; int ret;
if (memblk_nr_poison(mem)) return -EHWPOISON;
zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group, start_pfn, nr_pages);
/* * Although vmemmap pages have a different lifecycle than the pages * they describe (they remain until the memory is unplugged), doing * their initialization and accounting at memory onlining/offlining * stage helps to keep accounting easier to follow - e.g vmemmaps * belong to the same zone as the memory they backed. */ if (mem->altmap) nr_vmemmap_pages = mem->altmap->free;
mem_hotplug_begin(); clear_zone_contiguous(zone); if (nr_vmemmap_pages) { ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); if (ret) goto out; }
ret = online_pages(start_pfn + nr_vmemmap_pages, nr_pages - nr_vmemmap_pages, zone, mem->group); if (ret) { if (nr_vmemmap_pages) mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); goto out; }
/* * Account once onlining succeeded. If the zone was unpopulated, it is * now already properly populated. */ if (nr_vmemmap_pages) adjust_present_page_count(pfn_to_page(start_pfn), mem->group, nr_vmemmap_pages);
mem->zone = zone;out: set_zone_contiguous(zone); mem_hotplug_done(); return ret;}
/* * Must acquire mem_hotplug_lock in write mode. */static int memory_block_offline(struct memory_block *mem){ unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; unsigned long nr_vmemmap_pages = 0; int ret;
if (!mem->zone) return -EINVAL;
/* * Unaccount before offlining, such that unpopulated zone and kthreads * can properly be torn down in offline_pages(). */ if (mem->altmap) nr_vmemmap_pages = mem->altmap->free;
mem_hotplug_begin(); clear_zone_contiguous(mem->zone); if (nr_vmemmap_pages) adjust_present_page_count(pfn_to_page(start_pfn), mem->group, -nr_vmemmap_pages);
ret = offline_pages(start_pfn + nr_vmemmap_pages, nr_pages - nr_vmemmap_pages, mem->zone, mem->group); if (ret) { /* offline_pages() failed. Account back. */ if (nr_vmemmap_pages) adjust_present_page_count(pfn_to_page(start_pfn), mem->group, nr_vmemmap_pages); goto out; }
if (nr_vmemmap_pages) mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);
out: set_zone_contiguous(mem->zone); if (!ret) mem->zone = NULL; mem_hotplug_done(); return ret;}
/* * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is * OK to have direct references to sparsemem variables in here. */static intmemory_block_action(struct memory_block *mem, unsigned long action){ int ret;
switch (action) { case MEM_ONLINE: ret = memory_block_online(mem); break; case MEM_OFFLINE: ret = memory_block_offline(mem); break; default: WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: " "%ld\n", __func__, mem->start_section_nr, action, action); ret = -EINVAL; }
return ret;}
static int memory_block_change_state(struct memory_block *mem, unsigned long to_state, unsigned long from_state_req){ int ret = 0;
if (mem->state != from_state_req) return -EINVAL;
if (to_state == MEM_OFFLINE) mem->state = MEM_GOING_OFFLINE;
ret = memory_block_action(mem, to_state); mem->state = ret ? from_state_req : to_state;
return ret;}
/* The device lock serializes operations on memory_subsys_[online|offline] */static int memory_subsys_online(struct device *dev){ struct memory_block *mem = to_memory_block(dev); int ret;
if (mem->state == MEM_ONLINE) return 0;
/* * When called via device_online() without configuring the online_type, * we want to default to MMOP_ONLINE. */ if (mem->online_type == MMOP_OFFLINE) mem->online_type = MMOP_ONLINE;
ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); mem->online_type = MMOP_OFFLINE;
return ret;}
static int memory_subsys_offline(struct device *dev){ struct memory_block *mem = to_memory_block(dev);
if (mem->state == MEM_OFFLINE) return 0;
return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);}
static ssize_t state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count){ const int online_type = mhp_online_type_from_str(buf); struct memory_block *mem = to_memory_block(dev); int ret;
if (online_type < 0) return -EINVAL;
ret = lock_device_hotplug_sysfs(); if (ret) return ret;
switch (online_type) { case MMOP_ONLINE_KERNEL: case MMOP_ONLINE_MOVABLE: case MMOP_ONLINE: /* mem->online_type is protected by device_hotplug_lock */ mem->online_type = online_type; ret = device_online(&mem->dev); break; case MMOP_OFFLINE: ret = device_offline(&mem->dev); break; default: ret = -EINVAL; /* should never happen */ }
unlock_device_hotplug();
if (ret < 0) return ret; if (ret) return -EINVAL;
return count;}
/* * Legacy interface that we cannot remove: s390x exposes the storage increment * covered by a memory block, allowing for identifying which memory blocks * comprise a storage increment. Since a memory block spans complete * storage increments nowadays, this interface is basically unused. Other * archs never exposed != 0. */static ssize_t phys_device_show(struct device *dev, struct device_attribute *attr, char *buf){ struct memory_block *mem = to_memory_block(dev); unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
return sysfs_emit(buf, "%d\n", arch_get_memory_phys_device(start_pfn));}
#ifdef CONFIG_MEMORY_HOTREMOVEstatic int print_allowed_zone(char *buf, int len, int nid, struct memory_group *group, unsigned long start_pfn, unsigned long nr_pages, enum mmop online_type, struct zone *default_zone){ struct zone *zone;
zone = zone_for_pfn_range(online_type, nid, group, start_pfn, nr_pages); if (zone == default_zone) return 0;
return sysfs_emit_at(buf, len, " %s", zone->name);}
static ssize_t valid_zones_show(struct device *dev, struct device_attribute *attr, char *buf){ struct memory_block *mem = to_memory_block(dev); unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; struct memory_group *group = mem->group; struct zone *default_zone; int nid = mem->nid; int len;
/* * Check the existing zone. Make sure that we do that only on the * online nodes otherwise the page_zone is not reliable */ if (mem->state == MEM_ONLINE) { /* * If !mem->zone, the memory block spans multiple zones and * cannot get offlined. */ return sysfs_emit(buf, "%s\n", mem->zone ? mem->zone->name : "none"); }
default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, group, start_pfn, nr_pages);
len = sysfs_emit(buf, "%s", default_zone->name); len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages, MMOP_ONLINE_KERNEL, default_zone); len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages, MMOP_ONLINE_MOVABLE, default_zone); len += sysfs_emit_at(buf, len, "\n"); return len;}static DEVICE_ATTR_RO(valid_zones);#endif
static DEVICE_ATTR_RO(phys_index);static DEVICE_ATTR_RW(state);static DEVICE_ATTR_RO(phys_device);static DEVICE_ATTR_RO(removable);
/* * Show the memory block size (shared by all memory blocks). */static ssize_t block_size_bytes_show(struct device *dev, struct device_attribute *attr, char *buf){ return sysfs_emit(buf, "%lx\n", memory_block_size_bytes());}
static DEVICE_ATTR_RO(block_size_bytes);
/* * Memory auto online policy. */
static ssize_t auto_online_blocks_show(struct device *dev, struct device_attribute *attr, char *buf){ return sysfs_emit(buf, "%s\n", online_type_to_str[mhp_get_default_online_type()]);}
static ssize_t auto_online_blocks_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count){ const int online_type = mhp_online_type_from_str(buf);
if (online_type < 0) return -EINVAL;
mhp_set_default_online_type(online_type); return count;}
static DEVICE_ATTR_RW(auto_online_blocks);
#ifdef CONFIG_CRASH_HOTPLUG#include <linux/kexec.h>static ssize_t crash_hotplug_show(struct device *dev, struct device_attribute *attr, char *buf){ return sysfs_emit(buf, "%d\n", crash_check_hotplug_support());}static DEVICE_ATTR_RO(crash_hotplug);#endif
/* * Some architectures will have custom drivers to do this, and * will not need to do it from userspace. The fake hot-add code * as well as ppc64 will do all of their discovery in userspace * and will require this interface. */#ifdef CONFIG_ARCH_MEMORY_PROBEstatic ssize_t probe_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count){ u64 phys_addr; int nid, ret; unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
ret = kstrtoull(buf, 0, &phys_addr); if (ret) return ret;
if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) return -EINVAL;
ret = lock_device_hotplug_sysfs(); if (ret) return ret;
nid = memory_add_physaddr_to_nid(phys_addr); ret = __add_memory(nid, phys_addr, MIN_MEMORY_BLOCK_SIZE * sections_per_block, MHP_NONE);
if (ret) goto out;
ret = count;out: unlock_device_hotplug(); return ret;}
static DEVICE_ATTR_WO(probe);#endif
#ifdef CONFIG_MEMORY_FAILURE/* * Support for offlining pages of memory */
/* Soft offline a page */static ssize_t soft_offline_page_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count){ int ret; u64 pfn; if (!capable(CAP_SYS_ADMIN)) return -EPERM; if (kstrtoull(buf, 0, &pfn) < 0) return -EINVAL; pfn >>= PAGE_SHIFT; ret = soft_offline_page(pfn, 0); return ret == 0 ? count : ret;}
/* Forcibly offline a page, including killing processes. */static ssize_t hard_offline_page_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count){ int ret; u64 pfn; if (!capable(CAP_SYS_ADMIN)) return -EPERM; if (kstrtoull(buf, 0, &pfn) < 0) return -EINVAL; pfn >>= PAGE_SHIFT; ret = memory_failure(pfn, MF_SW_SIMULATED); if (ret == -EOPNOTSUPP) ret = 0; return ret ? ret : count;}
static DEVICE_ATTR_WO(soft_offline_page);static DEVICE_ATTR_WO(hard_offline_page);#endif
/* See phys_device_show(). */int __weak arch_get_memory_phys_device(unsigned long start_pfn){ return 0;}
/* * A reference for the returned memory block device is acquired. * * Called under device_hotplug_lock. */struct memory_block *memory_block_get(unsigned long block_id){ struct memory_block *mem;
mem = xa_load(&memory_blocks, block_id); if (mem) get_device(&mem->dev); return mem;}
static struct attribute *memory_memblk_attrs[] = { &dev_attr_phys_index.attr, &dev_attr_state.attr, &dev_attr_phys_device.attr, &dev_attr_removable.attr,#ifdef CONFIG_MEMORY_HOTREMOVE &dev_attr_valid_zones.attr,#endif NULL};
static const struct attribute_group memory_memblk_attr_group = { .attrs = memory_memblk_attrs,};
static const struct attribute_group *memory_memblk_attr_groups[] = { &memory_memblk_attr_group, NULL,};
static int __add_memory_block(struct memory_block *memory){ int ret;
memory->dev.bus = &memory_subsys; memory->dev.id = memory->start_section_nr / sections_per_block; memory->dev.release = memory_block_release; memory->dev.groups = memory_memblk_attr_groups; dev_assign_offline(&memory->dev, memory->state == MEM_OFFLINE);
ret = device_register(&memory->dev); if (ret) { memory_block_put(memory); return ret; } ret = xa_err(xa_store(&memory_blocks, memory->dev.id, memory, GFP_KERNEL)); if (ret) device_unregister(&memory->dev);
return ret;}
static struct zone *early_node_zone_for_memory_block(struct memory_block *mem, int nid){ const unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); const unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; struct zone *zone, *matching_zone = NULL; pg_data_t *pgdat = NODE_DATA(nid); int i;
/* * This logic only works for early memory, when the applicable zones * already span the memory block. We don't expect overlapping zones on * a single node for early memory. So if we're told that some PFNs * of a node fall into this memory block, we can assume that all node * zones that intersect with the memory block are actually applicable. * No need to look at the memmap. */ for (i = 0; i < MAX_NR_ZONES; i++) { zone = pgdat->node_zones + i; if (!populated_zone(zone)) continue; if (!zone_intersects(zone, start_pfn, nr_pages)) continue; if (!matching_zone) { matching_zone = zone; continue; } /* Spans multiple zones ... */ matching_zone = NULL; break; } return matching_zone;}
#ifdef CONFIG_NUMA/** * memory_block_add_nid_early() - Indicate that early system RAM falling into * this memory block device (partially) belongs * to the given node. * @mem: The memory block device. * @nid: The node id. * * Indicate that early system RAM falling into this memory block (partially) * belongs to the given node. This will also properly set/adjust mem->zone based * on the zone ranges of the given node. * * Memory hotplug handles this on memory block creation, where we can only have * a single nid span a memory block. */void memory_block_add_nid_early(struct memory_block *mem, int nid){ if (mem->nid != nid) { /* * For early memory we have to determine the zone when setting * the node id and handle multiple nodes spanning a single * memory block by indicate via zone == NULL that we're not * dealing with a single zone. So if we're setting the node id * the first time, determine if there is a single zone. If we're * setting the node id a second time to a different node, * invalidate the single detected zone. */ if (mem->nid == NUMA_NO_NODE) mem->zone = early_node_zone_for_memory_block(mem, nid); else mem->zone = NULL; /* * If this memory block spans multiple nodes, we only indicate * the last processed node. If we span multiple nodes (not applicable * to hotplugged memory), zone == NULL will prohibit memory offlining * and consequently unplug. */ mem->nid = nid; }}#endif
static int add_memory_block(unsigned long block_id, int nid, unsigned long state, struct vmem_altmap *altmap, struct memory_group *group){ struct memory_block *mem; int ret = 0;
mem = memory_block_get(block_id); if (mem) { memory_block_put(mem); return -EEXIST; } mem = kzalloc_obj(*mem); if (!mem) return -ENOMEM;
mem->start_section_nr = block_id * sections_per_block; mem->state = state; mem->nid = nid; INIT_LIST_HEAD(&mem->group_next);
#ifndef CONFIG_NUMA if (state == MEM_ONLINE) /* * MEM_ONLINE at this point implies early memory. With NUMA, * we'll determine the zone when setting the node id via * memory_block_add_nid_early(). Memory hotplug updated the zone * manually when memory onlining/offlining succeeds. */ mem->zone = early_node_zone_for_memory_block(mem, NUMA_NO_NODE);#endif /* CONFIG_NUMA */
ret = __add_memory_block(mem); if (ret) return ret;
mem->altmap = altmap;
if (group) { mem->group = group; list_add(&mem->group_next, &group->memory_blocks); }
return 0;}
static void remove_memory_block(struct memory_block *memory){ if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys)) return;
WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL);
if (memory->group) { list_del(&memory->group_next); memory->group = NULL; }
/* drop the ref. we got via memory_block_get() */ memory_block_put(memory); device_unregister(&memory->dev);}
/* * Create memory block devices for the given memory area. Start and size * have to be aligned to memory block granularity. Memory block devices * will be initialized as offline. * * Called under device_hotplug_lock. */int create_memory_block_devices(unsigned long start, unsigned long size, int nid, struct vmem_altmap *altmap, struct memory_group *group){ const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start)); unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size)); struct memory_block *mem; unsigned long block_id; int ret = 0;
if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) || !IS_ALIGNED(size, memory_block_size_bytes()))) return -EINVAL;
for (block_id = start_block_id; block_id != end_block_id; block_id++) { ret = add_memory_block(block_id, nid, MEM_OFFLINE, altmap, group); if (ret) break; } if (ret) { end_block_id = block_id; for (block_id = start_block_id; block_id != end_block_id; block_id++) { mem = memory_block_get(block_id); if (WARN_ON_ONCE(!mem)) continue; remove_memory_block(mem); } } return ret;}
/* * Remove memory block devices for the given memory area. Start and size * have to be aligned to memory block granularity. Memory block devices * have to be offline. * * Called under device_hotplug_lock. */void remove_memory_block_devices(unsigned long start, unsigned long size){ const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start)); const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size)); struct memory_block *mem; unsigned long block_id;
if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) || !IS_ALIGNED(size, memory_block_size_bytes()))) return;
for (block_id = start_block_id; block_id != end_block_id; block_id++) { mem = memory_block_get(block_id); if (WARN_ON_ONCE(!mem)) continue; num_poisoned_pages_sub(-1UL, memblk_nr_poison(mem)); unregister_memory_block_under_nodes(mem); remove_memory_block(mem); }}
static struct attribute *memory_root_attrs[] = {#ifdef CONFIG_ARCH_MEMORY_PROBE &dev_attr_probe.attr,#endif
#ifdef CONFIG_MEMORY_FAILURE &dev_attr_soft_offline_page.attr, &dev_attr_hard_offline_page.attr,#endif
&dev_attr_block_size_bytes.attr, &dev_attr_auto_online_blocks.attr,#ifdef CONFIG_CRASH_HOTPLUG &dev_attr_crash_hotplug.attr,#endif NULL};
static const struct attribute_group memory_root_attr_group = { .attrs = memory_root_attrs,};
static const struct attribute_group *memory_root_attr_groups[] = { &memory_root_attr_group, NULL,};
/* * Initialize the sysfs support for memory devices. At the time this function * is called, we cannot have concurrent creation/deletion of memory block * devices, the device_hotplug_lock is not needed. */void __init memory_dev_init(void){ int ret; unsigned long block_sz, block_id, nr;
/* Validate the configured memory block size */ block_sz = memory_block_size_bytes(); if (!is_power_of_2(block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE) panic("Memory block size not suitable: 0x%lx\n", block_sz); sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
ret = subsys_system_register(&memory_subsys, memory_root_attr_groups); if (ret) panic("%s() failed to register subsystem: %d\n", __func__, ret);
/* * Create entries for memory sections that were found during boot * and have been initialized. Use @block_id to track the last * handled block and initialize it to an invalid value (ULONG_MAX) * to bypass the block ID matching check for the first present * block so that it can be covered. */ block_id = ULONG_MAX; for_each_early_section_nr(0, nr) { if (block_id != ULONG_MAX && memory_block_id(nr) == block_id) continue;
block_id = memory_block_id(nr); ret = add_memory_block(block_id, NUMA_NO_NODE, MEM_ONLINE, NULL, NULL); if (ret) { panic("%s() failed to add memory block: %d\n", __func__, ret); } }}
/** * walk_memory_blocks - walk through all present memory blocks overlapped * by the range [start, start + size) * * @start: start address of the memory range * @size: size of the memory range * @arg: argument passed to func * @func: callback for each memory section walked * * This function walks through all present memory blocks overlapped by the * range [start, start + size), calling func on each memory block. * * In case func() returns an error, walking is aborted and the error is * returned. * * Called under device_hotplug_lock. */int walk_memory_blocks(unsigned long start, unsigned long size, void *arg, walk_memory_blocks_func_t func){ const unsigned long start_block_id = phys_to_block_id(start); const unsigned long end_block_id = phys_to_block_id(start + size - 1); struct memory_block *mem; unsigned long block_id; int ret = 0;
if (!size) return 0;
for (block_id = start_block_id; block_id <= end_block_id; block_id++) { mem = memory_block_get(block_id); if (!mem) continue;
ret = func(mem, arg); memory_block_put(mem); if (ret) break; } return ret;}
struct for_each_memory_block_cb_data { walk_memory_blocks_func_t func; void *arg;};
static int for_each_memory_block_cb(struct device *dev, void *data){ struct memory_block *mem = to_memory_block(dev); struct for_each_memory_block_cb_data *cb_data = data;
return cb_data->func(mem, cb_data->arg);}
/** * for_each_memory_block - walk through all present memory blocks * * @arg: argument passed to func * @func: callback for each memory block walked * * This function walks through all present memory blocks, calling func on * each memory block. * * In case func() returns an error, walking is aborted and the error is * returned. */int for_each_memory_block(void *arg, walk_memory_blocks_func_t func){ struct for_each_memory_block_cb_data cb_data = { .func = func, .arg = arg, };
return bus_for_each_dev(&memory_subsys, NULL, &cb_data, for_each_memory_block_cb);}
/* * This is an internal helper to unify allocation and initialization of * memory groups. Note that the passed memory group will be copied to a * dynamically allocated memory group. After this call, the passed * memory group should no longer be used. */static int memory_group_register(struct memory_group group){ struct memory_group *new_group; uint32_t mgid; int ret;
if (!node_possible(group.nid)) return -EINVAL;
new_group = kzalloc_obj(group); if (!new_group) return -ENOMEM; *new_group = group; INIT_LIST_HEAD(&new_group->memory_blocks);
ret = xa_alloc(&memory_groups, &mgid, new_group, xa_limit_31b, GFP_KERNEL); if (ret) { kfree(new_group); return ret; } else if (group.is_dynamic) { xa_set_mark(&memory_groups, mgid, MEMORY_GROUP_MARK_DYNAMIC); } return mgid;}
/** * memory_group_register_static() - Register a static memory group. * @nid: The node id. * @max_pages: The maximum number of pages we'll have in this static memory * group. * * Register a new static memory group and return the memory group id. * All memory in the group belongs to a single unit, such as a DIMM. All * memory belonging to a static memory group is added in one go to be removed * in one go -- it's static. * * Returns an error if out of memory, if the node id is invalid, if no new * memory groups can be registered, or if max_pages is invalid (0). Otherwise, * returns the new memory group id. */int memory_group_register_static(int nid, unsigned long max_pages){ struct memory_group group = { .nid = nid, .s = { .max_pages = max_pages, }, };
if (!max_pages) return -EINVAL; return memory_group_register(group);}EXPORT_SYMBOL_GPL(memory_group_register_static);
/** * memory_group_register_dynamic() - Register a dynamic memory group. * @nid: The node id. * @unit_pages: Unit in pages in which is memory added/removed in this dynamic * memory group. * * Register a new dynamic memory group and return the memory group id. * Memory within a dynamic memory group is added/removed dynamically * in unit_pages. * * Returns an error if out of memory, if the node id is invalid, if no new * memory groups can be registered, or if unit_pages is invalid (0, not a * power of two, smaller than a single memory block). Otherwise, returns the * new memory group id. */int memory_group_register_dynamic(int nid, unsigned long unit_pages){ struct memory_group group = { .nid = nid, .is_dynamic = true, .d = { .unit_pages = unit_pages, }, };
if (!unit_pages || !is_power_of_2(unit_pages) || unit_pages < PHYS_PFN(memory_block_size_bytes())) return -EINVAL; return memory_group_register(group);}EXPORT_SYMBOL_GPL(memory_group_register_dynamic);
/** * memory_group_unregister() - Unregister a memory group. * @mgid: the memory group id * * Unregister a memory group. If any memory block still belongs to this * memory group, unregistering will fail. * * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some * memory blocks still belong to this memory group and returns 0 if * unregistering succeeded. */int memory_group_unregister(int mgid){ struct memory_group *group;
if (mgid < 0) return -EINVAL;
group = xa_load(&memory_groups, mgid); if (!group) return -EINVAL; if (!list_empty(&group->memory_blocks)) return -EBUSY; xa_erase(&memory_groups, mgid); kfree(group); return 0;}EXPORT_SYMBOL_GPL(memory_group_unregister);
/* * This is an internal helper only to be used in core memory hotplug code to * lookup a memory group. We don't care about locking, as we don't expect a * memory group to get unregistered while adding memory to it -- because * the group and the memory is managed by the same driver. */struct memory_group *memory_group_find_by_id(int mgid){ return xa_load(&memory_groups, mgid);}
/* * This is an internal helper only to be used in core memory hotplug code to * walk all dynamic memory groups excluding a given memory group, either * belonging to a specific node, or belonging to any node. */int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func, struct memory_group *excluded, void *arg){ struct memory_group *group; unsigned long index; int ret = 0;
xa_for_each_marked(&memory_groups, index, group, MEMORY_GROUP_MARK_DYNAMIC) { if (group == excluded) continue;#ifdef CONFIG_NUMA if (nid != NUMA_NO_NODE && group->nid != nid) continue;#endif /* CONFIG_NUMA */ ret = func(group, arg); if (ret) break; } return ret;}
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)void memblk_nr_poison_inc(unsigned long pfn){ const unsigned long block_id = pfn_to_block_id(pfn); struct memory_block *mem = memory_block_get(block_id);
if (mem) { atomic_long_inc(&mem->nr_hwpoison); memory_block_put(mem); }}
void memblk_nr_poison_sub(unsigned long pfn, long i){ const unsigned long block_id = pfn_to_block_id(pfn); struct memory_block *mem = memory_block_get(block_id);
if (mem) { atomic_long_sub(i, &mem->nr_hwpoison); memory_block_put(mem); }}
static unsigned long memblk_nr_poison(struct memory_block *mem){ return atomic_long_read(&mem->nr_hwpoison);}#endif