Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289// SPDX-License-Identifier: GPL-2.0/* * sparse memory mappings. */#include <linux/mm.h>#include <linux/slab.h>#include <linux/mmzone.h>#include <linux/memblock.h>#include <linux/compiler.h>#include <linux/highmem.h>#include <linux/export.h>#include <linux/spinlock.h>#include <linux/vmalloc.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/vmstat.h>#include "internal.h"#include "mm_init.h"#include "sparse.h"#include <asm/dma.h>
/* * Permanent SPARSEMEM data: * * 1) mem_section - memory sections, mem_map's for valid memory */#ifdef CONFIG_SPARSEMEM_EXTREMEstruct mem_section **mem_section;#elsestruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] ____cacheline_internodealigned_in_smp;#endifEXPORT_SYMBOL(mem_section);
#ifdef NODE_NOT_IN_PAGE_FLAGS/* * If we did not store the node number in the page then we have to * do a lookup in the section_to_node_table in order to find which * node the page belongs to. */#if MAX_NUMNODES <= 256static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#elsestatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#endif
int memdesc_nid(const memdesc_flags_t *mdf){ return section_to_node_table[memdesc_section(mdf)];}EXPORT_SYMBOL(memdesc_nid);
static void set_section_nid(unsigned long section_nr, int nid){ section_to_node_table[section_nr] = nid;}#else /* !NODE_NOT_IN_PAGE_FLAGS */static inline void set_section_nid(unsigned long section_nr, int nid){}#endif
#ifdef CONFIG_SPARSEMEM_EXTREMEstatic noinline struct mem_section __ref *sparse_index_alloc(int nid){ struct mem_section *section = NULL; unsigned long array_size = SECTIONS_PER_ROOT * sizeof(struct mem_section);
if (slab_is_available()) { section = kzalloc_node(array_size, GFP_KERNEL, nid); } else { section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, nid); if (!section) panic("%s: Failed to allocate %lu bytes nid=%d\n", __func__, array_size, nid); }
return section;}
int __meminit sparse_index_init(unsigned long section_nr, int nid){ unsigned long root = SECTION_NR_TO_ROOT(section_nr); struct mem_section *section;
/* * An existing section is possible in the sub-section hotplug * case. First hot-add instantiates, follow-on hot-add reuses * the existing section. * * The mem_hotplug_lock resolves the apparent race below. */ if (mem_section[root]) return 0;
section = sparse_index_alloc(nid); if (!section) return -ENOMEM;
mem_section[root] = section;
return 0;}
static void __init sparse_extreme_init(void){ const unsigned long size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
mem_section = memblock_alloc_or_panic(size, INTERNODE_CACHE_BYTES);}#else /* !SPARSEMEM_EXTREME */int __meminit sparse_index_init(unsigned long section_nr, int nid){ return 0;}
static void __init sparse_extreme_init(void){}#endif
/* * During early boot, before section_mem_map is used for an actual * mem_map, we use section_mem_map to store the section's NUMA * node. This keeps us from having to use another data structure. The * node information is cleared just before we store the real mem_map. */static inline unsigned long sparse_encode_early_nid(int nid){ return ((unsigned long)nid << SECTION_NID_SHIFT);}
static inline int sparse_early_nid(struct mem_section *section){ return (section->section_mem_map >> SECTION_NID_SHIFT);}
/* Validate the physical addressing limitations of the model */static void __init mminit_validate_memmodel_limits(unsigned long *start_pfn, unsigned long *end_pfn){ unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT;
/* * Sanity checks - do not allow an architecture to pass * in larger pfns than the maximum scope of sparsemem: */ if (*start_pfn > max_sparsemem_pfn) { mminit_dprintk(MMINIT_WARNING, "pfnvalidation", "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", *start_pfn, *end_pfn, max_sparsemem_pfn); WARN_ON_ONCE(1); *start_pfn = max_sparsemem_pfn; *end_pfn = max_sparsemem_pfn; } else if (*end_pfn > max_sparsemem_pfn) { mminit_dprintk(MMINIT_WARNING, "pfnvalidation", "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", *start_pfn, *end_pfn, max_sparsemem_pfn); WARN_ON_ONCE(1); *end_pfn = max_sparsemem_pfn; }}
/* * Looping over all possible memory sections is expensive, especially if * NR_MEM_SECTIONS is large but only a fraction is actually used. Keep track of * the highest section number we ever used. */unsigned long __highest_used_section_nr;
static inline unsigned long first_early_section_nr(void){ return next_early_section_nr(-1);}
void __init sparse_sections_init(void){ const unsigned long flags = SECTION_IS_EARLY | SECTION_IS_ONLINE; unsigned long pfn, start_pfn, end_pfn, section_nr; int i, nid;
sparse_extreme_init();
for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { start_pfn &= PAGE_SECTION_MASK; mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { struct mem_section *ms;
section_nr = pfn_to_section_nr(pfn); sparse_index_init(section_nr, nid); ms = __nr_to_section(section_nr); if (ms->section_mem_map) continue;
set_section_nid(section_nr, nid); ms->section_mem_map = sparse_encode_early_nid(nid) | flags; } } __highest_used_section_nr = section_nr;}#ifndef CONFIG_SPARSEMEM_VMEMMAPstruct page __init *__populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap){ const unsigned long size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid);}#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
void __weak __meminit vmemmap_populate_print_last(void){}
static void __init sparse_metadata_init_nid(int nid, unsigned long start_section_nr, unsigned long end_section_nr, unsigned long nr_sections){ struct mem_section_usage *usage; unsigned long section_nr;
usage = memblock_alloc_node(nr_sections * mem_section_usage_size(), SMP_CACHE_BYTES, nid); if (!usage) panic("Failed to allocate usemap for node %d\n", nid);
for_each_early_section_nr(start_section_nr, section_nr) { const unsigned long pfn = section_nr_to_pfn(section_nr); struct page *mem_map;
if (section_nr >= end_section_nr) break;
mem_map = __populate_section_memmap(pfn, PAGES_PER_SECTION, nid, NULL, NULL); if (!mem_map) panic("Failed to allocate memmap for section %lu\n", section_nr); memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION)); sparse_init_one_section(__nr_to_section(section_nr), section_nr, mem_map, usage); usage = (void *)usage + mem_section_usage_size(); }}
static void __init sparse_metadata_init(void){ unsigned long start_section_nr = first_early_section_nr(); int nid_begin = sparse_early_nid(__nr_to_section(start_section_nr)); unsigned long section_nr, nr_sections = 1;
for_each_early_section_nr(start_section_nr + 1, section_nr) { const int nid = sparse_early_nid(__nr_to_section(section_nr));
if (nid == nid_begin) { nr_sections++; continue; } sparse_metadata_init_nid(nid_begin, start_section_nr, section_nr, nr_sections); nid_begin = nid; start_section_nr = section_nr; nr_sections = 1; } sparse_metadata_init_nid(nid_begin, start_section_nr, section_nr, nr_sections);}
/* * Allocate the accumulated non-linear sections, allocate a mem_map * for each and record the physical to section mapping. */void __init sparse_init(void){ if (compound_info_has_mask()) { VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN)); }
sparse_metadata_init(); sparse_init_subsection_map(); vmemmap_populate_print_last();}