Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690// SPDX-License-Identifier: GPL-2.0-only/* * linux/init/main.c * * Copyright (C) 1991, 1992 Linus Torvalds * * GK 2/5/95 - Changed to support mounting root fs via NFS * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96 * Simplified starting of init: Michael A. Griffith <grif@acm.org> */
#define DEBUG /* Enable initcall_debug */
#include <linux/types.h>#include <linux/export.h>#include <linux/extable.h>#include <linux/module.h>#include <linux/proc_fs.h>#include <linux/binfmts.h>#include <linux/kernel.h>#include <linux/syscalls.h>#include <linux/stackprotector.h>#include <linux/string.h>#include <linux/ctype.h>#include <linux/delay.h>#include <linux/ioport.h>#include <linux/init.h>#include <linux/initrd.h>#include <linux/memblock.h>#include <linux/acpi.h>#include <linux/bootconfig.h>#include <linux/console.h>#include <linux/nmi.h>#include <linux/percpu.h>#include <linux/kmod.h>#include <linux/kprobes.h>#include <linux/kmsan.h>#include <linux/ksysfs.h>#include <linux/vmalloc.h>#include <linux/kernel_stat.h>#include <linux/start_kernel.h>#include <linux/security.h>#include <linux/smp.h>#include <linux/profile.h>#include <linux/kfence.h>#include <linux/rcupdate.h>#include <linux/srcu.h>#include <linux/moduleparam.h>#include <linux/kallsyms.h>#include <linux/buildid.h>#include <linux/writeback.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/memcontrol.h>#include <linux/cgroup.h>#include <linux/tick.h>#include <linux/sched/isolation.h>#include <linux/interrupt.h>#include <linux/taskstats_kern.h>#include <linux/delayacct.h>#include <linux/unistd.h>#include <linux/utsname.h>#include <linux/rmap.h>#include <linux/mempolicy.h>#include <linux/key.h>#include <linux/debug_locks.h>#include <linux/debugobjects.h>#include <linux/lockdep.h>#include <linux/kmemleak.h>#include <linux/padata.h>#include <linux/pid_namespace.h>#include <linux/device/driver.h>#include <linux/kthread.h>#include <linux/sched.h>#include <linux/sched/init.h>#include <linux/signal.h>#include <linux/idr.h>#include <linux/kgdb.h>#include <linux/ftrace.h>#include <linux/async.h>#include <linux/shmem_fs.h>#include <linux/slab.h>#include <linux/perf_event.h>#include <linux/ptrace.h>#include <linux/pti.h>#include <linux/blkdev.h>#include <linux/sched/clock.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/context_tracking.h>#include <linux/random.h>#include <linux/moduleloader.h>#include <linux/list.h>#include <linux/integrity.h>#include <linux/proc_ns.h>#include <linux/io.h>#include <linux/cache.h>#include <linux/rodata_test.h>#include <linux/jump_label.h>#include <linux/kcsan.h>#include <linux/init_syscalls.h>#include <linux/stackdepot.h>#include <linux/randomize_kstack.h>#include <linux/pidfs.h>#include <linux/ptdump.h>#include <linux/time_namespace.h>#include <linux/unaligned.h>#include <linux/vdso_datastore.h>#include <net/net_namespace.h>
#include <asm/io.h>#include <asm/setup.h>#include <asm/sections.h>#include <asm/cacheflush.h>
#define CREATE_TRACE_POINTS#include <trace/events/initcall.h>
#include <kunit/test.h>
static int kernel_init(void *);
/* * Debug helper: via this flag we know that we are in 'early bootup code' * where only the boot processor is running with IRQ disabled. This means * two things - IRQ must not be enabled before the flag is cleared and some * operations which are not allowed with IRQ disabled are allowed while the * flag is set. */bool early_boot_irqs_disabled __read_mostly;
enum system_states system_state __read_mostly;EXPORT_SYMBOL(system_state);
/* * Boot command-line arguments */#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
/* Default late time init is NULL. archs can override this later. */void (*__initdata late_time_init)(void);
/* Untouched command line saved by arch-specific code. */char __initdata boot_command_line[COMMAND_LINE_SIZE];/* Untouched saved command line (eg. for /proc) */char *saved_command_line __ro_after_init;unsigned int saved_command_line_len __ro_after_init;/* Command line for parameter parsing */static char *static_command_line;/* Untouched extra command line */static char *extra_command_line;/* Extra init arguments */static char *extra_init_args;
#ifdef CONFIG_BOOT_CONFIG/* Is bootconfig on command line? */static bool bootconfig_found;static size_t initargs_offs;#else# define bootconfig_found false# define initargs_offs 0#endif
static char *execute_command;static char *ramdisk_execute_command = "/init";static bool __initdata ramdisk_execute_command_set;
/* * Used to generate warnings if static_key manipulation functions are used * before jump_label_init is called. */bool static_key_initialized __read_mostly;EXPORT_SYMBOL_GPL(static_key_initialized);
/* * If set, this is an indication to the drivers that reset the underlying * device before going ahead with the initialization otherwise driver might * rely on the BIOS and skip the reset operation. * * This is useful if kernel is booting in an unreliable environment. * For ex. kdump situation where previous kernel has crashed, BIOS has been * skipped and devices will be in unknown state. */unsigned int reset_devices;EXPORT_SYMBOL(reset_devices);
static int __init set_reset_devices(char *str){ reset_devices = 1; return 1;}
__setup("reset_devices", set_reset_devices);
static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };static const char *panic_later, *panic_param;
static bool __init obsolete_checksetup(char *line){ const struct obs_kernel_param *p; bool had_early_param = false;
p = __setup_start; do { int n = strlen(p->str); if (parameqn(line, p->str, n)) { if (p->early) { /* Already done in parse_early_param? * (Needs exact match on param part). * Keep iterating, as we can have early * params and __setups of same names 8( */ if (line[n] == '\0' || line[n] == '=') had_early_param = true; } else if (!p->setup_func) { pr_warn("Parameter %s is obsolete, ignored\n", p->str); return true; } else if (p->setup_func(line + n)) return true; } p++; } while (p < __setup_end);
return had_early_param;}
/* * This should be approx 2 Bo*oMips to start (note initial shift), and will * still work even if initially too large, it will just take slightly longer */unsigned long loops_per_jiffy = (1<<12);EXPORT_SYMBOL(loops_per_jiffy);
static int __init debug_kernel(char *str){ console_loglevel = CONSOLE_LOGLEVEL_DEBUG; return 0;}
static int __init quiet_kernel(char *str){ console_loglevel = CONSOLE_LOGLEVEL_QUIET; return 0;}
early_param("debug", debug_kernel);early_param("quiet", quiet_kernel);
static int __init loglevel(char *str){ int newlevel;
/* * Only update loglevel value when a correct setting was passed, * to prevent blind crashes (when loglevel being set to 0) that * are quite hard to debug */ if (get_option(&str, &newlevel)) { console_loglevel = newlevel; return 0; }
return -EINVAL;}
early_param("loglevel", loglevel);
#ifdef CONFIG_BLK_DEV_INITRDstatic void * __init get_boot_config_from_initrd(size_t *_size){ u32 size, csum; char *data; u8 *hdr; int i;
if (!initrd_end) return NULL;
data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN; /* * Since Grub may align the size of initrd to 4, we must * check the preceding 3 bytes as well. */ for (i = 0; i < 4; i++) { if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN)) goto found; data--; } return NULL;
found: hdr = (u8 *)(data - 8); size = get_unaligned_le32(hdr); csum = get_unaligned_le32(hdr + 4);
data = ((void *)hdr) - size; if ((unsigned long)data < initrd_start) { pr_err("bootconfig size %d is greater than initrd size %ld\n", size, initrd_end - initrd_start); return NULL; }
if (xbc_calc_checksum(data, size) != csum) { pr_err("bootconfig checksum failed\n"); return NULL; }
/* Remove bootconfig from initramfs/initrd */ initrd_end = (unsigned long)data; if (_size) *_size = size;
return data;}#elsestatic void * __init get_boot_config_from_initrd(size_t *_size){ return NULL;}#endif
#ifdef CONFIG_BOOT_CONFIG
/* Make an extra command line under given key word */static char * __init xbc_make_cmdline(const char *key){ struct xbc_node *root; char *new_cmdline; int ret, len = 0;
root = xbc_find_node(key); if (!root) return NULL;
/* Count required buffer size */ len = xbc_snprint_cmdline(NULL, 0, root); if (len <= 0) return NULL;
new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES); if (!new_cmdline) { pr_err("Failed to allocate memory for extra kernel cmdline.\n"); return NULL; }
ret = xbc_snprint_cmdline(new_cmdline, len + 1, root); if (ret < 0 || ret > len) { pr_err("Failed to print extra kernel cmdline.\n"); memblock_free(new_cmdline, len + 1); return NULL; }
return new_cmdline;}
static int __init bootconfig_params(char *param, char *val, const char *unused, void *arg){ if (strcmp(param, "bootconfig") == 0) { bootconfig_found = true; } return 0;}
static int __init warn_bootconfig(char *str){ /* The 'bootconfig' has been handled by bootconfig_params(). */ return 0;}
static void __init setup_boot_config(void){ static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; const char *msg, *data; int pos, ret; size_t size; char *err;
/* Cut out the bootconfig data even if we have no bootconfig option */ data = get_boot_config_from_initrd(&size); /* If there is no bootconfig in initrd, try embedded one. */ if (!data) data = xbc_get_embedded_bootconfig(&size);
strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL, bootconfig_params);
if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE))) return;
/* parse_args() stops at the next param of '--' and returns an address */ if (err) initargs_offs = err - tmp_cmdline;
if (!data) { /* If user intended to use bootconfig, show an error level message */ if (bootconfig_found) pr_err("'bootconfig' found on command line, but no bootconfig found\n"); else pr_info("No bootconfig data provided, so skipping bootconfig"); return; }
if (size >= XBC_DATA_MAX) { pr_err("bootconfig size %ld greater than max size %d\n", (long)size, XBC_DATA_MAX); return; }
ret = xbc_init(data, size, &msg, &pos); if (ret < 0) { if (pos < 0) pr_err("Failed to init bootconfig: %s.\n", msg); else pr_err("Failed to parse bootconfig: %s at %d.\n", msg, pos); } else { xbc_get_info(&ret, NULL); pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret); /* keys starting with "kernel." are passed via cmdline */ extra_command_line = xbc_make_cmdline("kernel"); /* Also, "init." keys are init arguments */ extra_init_args = xbc_make_cmdline("init"); } return;}
static void __init exit_boot_config(void){ xbc_exit();}
#else /* !CONFIG_BOOT_CONFIG */
static void __init setup_boot_config(void){ /* Remove bootconfig data from initrd */ get_boot_config_from_initrd(NULL);}
static int __init warn_bootconfig(char *str){ pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n"); return 0;}
#define exit_boot_config() do {} while (0)
#endif /* CONFIG_BOOT_CONFIG */
early_param("bootconfig", warn_bootconfig);
bool __init cmdline_has_extra_options(void){ return extra_command_line || extra_init_args;}
/* Change NUL term back to "=", to make "param" the whole string. */static void __init repair_env_string(char *param, char *val){ if (val) { /* param=val or param="val"? */ if (val == param+strlen(param)+1) val[-1] = '='; else if (val == param+strlen(param)+2) { val[-2] = '='; memmove(val-1, val, strlen(val)+1); } else BUG(); }}
/* Anything after -- gets handed straight to init. */static int __init set_init_arg(char *param, char *val, const char *unused, void *arg){ unsigned int i;
if (panic_later) return 0;
repair_env_string(param, val);
for (i = 0; argv_init[i]; i++) { if (i == MAX_INIT_ARGS) { panic_later = "init"; panic_param = param; return 0; } } argv_init[i] = param; return 0;}
/* * Unknown boot options get handed to init, unless they look like * unused parameters (modprobe will find them in /proc/cmdline). */static int __init unknown_bootoption(char *param, char *val, const char *unused, void *arg){ size_t len = strlen(param); /* * Well-known bootloader identifiers: * 1. LILO/Grub pass "BOOT_IMAGE=..."; * 2. kexec/kdump (kexec-tools) pass "kexec". */ const char *bootloader[] = { "BOOT_IMAGE=", "kexec", NULL };
/* Handle params aliased to sysctls */ if (sysctl_is_alias(param)) return 0;
repair_env_string(param, val);
/* Handle bootloader identifier */ for (int i = 0; bootloader[i]; i++) { if (strstarts(param, bootloader[i])) return 0; }
/* Handle obsolete-style parameters */ if (obsolete_checksetup(param)) return 0;
/* Unused module parameter. */ if (strnchr(param, len, '.')) return 0;
if (panic_later) return 0;
if (val) { /* Environment option */ unsigned int i; for (i = 0; envp_init[i]; i++) { if (i == MAX_INIT_ENVS) { panic_later = "env"; panic_param = param; } if (!strncmp(param, envp_init[i], len+1)) break; } envp_init[i] = param; } else { /* Command line option */ unsigned int i; for (i = 0; argv_init[i]; i++) { if (i == MAX_INIT_ARGS) { panic_later = "init"; panic_param = param; } } argv_init[i] = param; } return 0;}
static int __init init_setup(char *str){ unsigned int i;
execute_command = str; /* * In case LILO is going to boot us with default command line, * it prepends "auto" before the whole cmdline which makes * the shell think it should execute a script with such name. * So we ignore all arguments entered _before_ init=... [MJ] */ for (i = 1; i < MAX_INIT_ARGS; i++) argv_init[i] = NULL; return 1;}__setup("init=", init_setup);
static int __init rdinit_setup(char *str){ unsigned int i;
ramdisk_execute_command = str; ramdisk_execute_command_set = true; /* See "auto" comment in init_setup */ for (i = 1; i < MAX_INIT_ARGS; i++) argv_init[i] = NULL; return 1;}__setup("rdinit=", rdinit_setup);
#ifndef CONFIG_SMPstatic inline void setup_nr_cpu_ids(void) { }static inline void smp_prepare_cpus(unsigned int maxcpus) { }#endif
/* * We need to store the untouched command line for future reference. * We also need to store the touched command line since the parameter * parsing is performed in place, and we should allow a component to * store reference of name/value for future reference. */static void __init setup_command_line(char *command_line){ size_t len, xlen = 0, ilen = 0;
if (extra_command_line) xlen = strlen(extra_command_line); if (extra_init_args) { extra_init_args = strim(extra_init_args); /* remove trailing space */ ilen = strlen(extra_init_args) + 4; /* for " -- " */ }
len = xlen + strlen(boot_command_line) + ilen + 1;
saved_command_line = memblock_alloc_or_panic(len, SMP_CACHE_BYTES);
len = xlen + strlen(command_line) + 1;
static_command_line = memblock_alloc_or_panic(len, SMP_CACHE_BYTES);
if (xlen) { /* * We have to put extra_command_line before boot command * lines because there could be dashes (separator of init * command line) in the command lines. */ strcpy(saved_command_line, extra_command_line); strcpy(static_command_line, extra_command_line); } strcpy(saved_command_line + xlen, boot_command_line); strcpy(static_command_line + xlen, command_line);
if (ilen) { /* * Append supplemental init boot args to saved_command_line * so that user can check what command line options passed * to init. * The order should always be * " -- "[bootconfig init-param][cmdline init-param] */ if (initargs_offs) { len = xlen + initargs_offs; strcpy(saved_command_line + len, extra_init_args); len += ilen - 4; /* strlen(extra_init_args) */ strcpy(saved_command_line + len, boot_command_line + initargs_offs - 1); } else { len = strlen(saved_command_line); strcpy(saved_command_line + len, " -- "); len += 4; strcpy(saved_command_line + len, extra_init_args); } }
saved_command_line_len = strlen(saved_command_line);}
/* * We need to finalize in a non-__init function or else race conditions * between the root thread and the init thread may cause start_kernel to * be reaped by free_initmem before the root thread has proceeded to * cpu_idle. * * gcc-3.4 accidentally inlines this function, so use noinline. */
static __initdata DECLARE_COMPLETION(kthreadd_done);
static noinline void __ref __noreturn rest_init(void){ struct task_struct *tsk; int pid;
rcu_scheduler_starting(); /* * We need to spawn init first so that it obtains pid 1, however * the init task will end up wanting to create kthreads, which, if * we schedule it before we create kthreadd, will OOPS. */ pid = user_mode_thread(kernel_init, NULL, CLONE_FS); /* * Pin init on the boot CPU. Task migration is not properly working * until sched_init_smp() has been run. It will set the allowed * CPUs for init to the non isolated CPUs. */ rcu_read_lock(); tsk = find_task_by_pid_ns(pid, &init_pid_ns); tsk->flags |= PF_NO_SETAFFINITY; set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id())); rcu_read_unlock();
numa_default_policy(); pid = kernel_thread(kthreadd, NULL, NULL, CLONE_FS | CLONE_FILES); rcu_read_lock(); kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns); rcu_read_unlock();
/* * Enable might_sleep() and smp_processor_id() checks. * They cannot be enabled earlier because with CONFIG_PREEMPTION=y * kernel_thread() would trigger might_sleep() splats. With * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled * already, but it's stuck on the kthreadd_done completion. */ system_state = SYSTEM_SCHEDULING;
complete(&kthreadd_done);
/* * The boot idle thread must execute schedule() * at least once to get things moving: */ schedule_preempt_disabled(); /* Call into cpu_idle with preempt disabled */ cpu_startup_entry(CPUHP_ONLINE);}
/* Check for early params. */static int __init do_early_param(char *param, char *val, const char *unused, void *arg){ const struct obs_kernel_param *p;
for (p = __setup_start; p < __setup_end; p++) { if (p->early && parameq(param, p->str)) { if (p->setup_func(val) != 0) pr_warn("Malformed early option '%s'\n", param); } } /* We accept everything at this stage. */ return 0;}
void __init parse_early_options(char *cmdline){ parse_args("early options", cmdline, NULL, 0, 0, 0, NULL, do_early_param);}
/* Arch code calls this early on, or if not, just before other parsing. */void __init parse_early_param(void){ static int done __initdata; static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
if (done) return;
/* All fall through to do_early_param. */ strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); parse_early_options(tmp_cmdline); done = 1;}
void __init __weak arch_post_acpi_subsys_init(void) { }
void __init __weak smp_setup_processor_id(void){}
void __init __weak smp_prepare_boot_cpu(void){}
# if THREAD_SIZE >= PAGE_SIZEvoid __init __weak thread_stack_cache_init(void){}#endif
void __init __weak poking_init(void) { }
void __init __weak pgtable_cache_init(void) { }
void __init __weak trap_init(void) { }
bool initcall_debug;core_param(initcall_debug, initcall_debug, bool, 0644);
#ifdef TRACEPOINTS_ENABLEDstatic void __init initcall_debug_enable(void);#elsestatic inline void initcall_debug_enable(void){}#endif
#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSETDEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, randomize_kstack_offset);DEFINE_PER_CPU(struct rnd_state, kstack_rnd_state);
static int __init random_kstack_init(void){ prandom_seed_full_state(&kstack_rnd_state); return 0;}late_initcall(random_kstack_init);
static int __init early_randomize_kstack_offset(char *buf){ int ret; bool bool_result;
ret = kstrtobool(buf, &bool_result); if (ret) return ret;
if (bool_result) static_branch_enable(&randomize_kstack_offset); else static_branch_disable(&randomize_kstack_offset); return 0;}early_param("randomize_kstack_offset", early_randomize_kstack_offset);#endif
static void __init print_unknown_bootoptions(void){ char *unknown_options; char *end; const char *const *p; size_t len;
if (panic_later || (!argv_init[1] && !envp_init[2])) return;
/* * Determine how many options we have to print out, plus a space * before each */ len = 1; /* null terminator */ for (p = &argv_init[1]; *p; p++) { len++; len += strlen(*p); } for (p = &envp_init[2]; *p; p++) { len++; len += strlen(*p); }
unknown_options = memblock_alloc(len, SMP_CACHE_BYTES); if (!unknown_options) { pr_err("%s: Failed to allocate %zu bytes\n", __func__, len); return; } end = unknown_options;
for (p = &argv_init[1]; *p; p++) end += sprintf(end, " %s", *p); for (p = &envp_init[2]; *p; p++) end += sprintf(end, " %s", *p);
/* Start at unknown_options[1] to skip the initial space */ pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n", &unknown_options[1]); memblock_free(unknown_options, len);}
static void __init early_numa_node_init(void){#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID#ifndef cpu_to_node int cpu;
/* The early_cpu_to_node() should be ready here. */ for_each_possible_cpu(cpu) set_cpu_numa_node(cpu, early_cpu_to_node(cpu));#endif#endif}
#define KERNEL_CMDLINE_PREFIX "Kernel command line: "#define KERNEL_CMDLINE_PREFIX_LEN (sizeof(KERNEL_CMDLINE_PREFIX) - 1)#define KERNEL_CMDLINE_CONTINUATION " \\"#define KERNEL_CMDLINE_CONTINUATION_LEN (sizeof(KERNEL_CMDLINE_CONTINUATION) - 1)
#define MIN_CMDLINE_LOG_WRAP_IDEAL_LEN (KERNEL_CMDLINE_PREFIX_LEN + \ KERNEL_CMDLINE_CONTINUATION_LEN)#define CMDLINE_LOG_WRAP_IDEAL_LEN (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN > \ MIN_CMDLINE_LOG_WRAP_IDEAL_LEN ? \ CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN : \ MIN_CMDLINE_LOG_WRAP_IDEAL_LEN)
#define IDEAL_CMDLINE_LEN (CMDLINE_LOG_WRAP_IDEAL_LEN - KERNEL_CMDLINE_PREFIX_LEN)#define IDEAL_CMDLINE_SPLIT_LEN (IDEAL_CMDLINE_LEN - KERNEL_CMDLINE_CONTINUATION_LEN)
/** * print_kernel_cmdline() - Print the kernel cmdline with wrapping. * @cmdline: The cmdline to print. * * Print the kernel command line, trying to wrap based on the Kconfig knob * CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN. * * Wrapping is based on spaces, ignoring quotes. All lines are prefixed * with "Kernel command line: " and lines that are not the last line have * a " \" suffix added to them. The prefix and suffix count towards the * line length for wrapping purposes. The ideal length will be exceeded * if no appropriate place to wrap is found. * * Example output if CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN is 40: * Kernel command line: loglevel=7 \ * Kernel command line: init=/sbin/init \ * Kernel command line: root=PARTUUID=8c3efc1a-768b-6642-8d0c-89eb782f19f0/PARTNROFF=1 \ * Kernel command line: rootwait ro \ * Kernel command line: my_quoted_arg="The \ * Kernel command line: quick brown fox \ * Kernel command line: jumps over the \ * Kernel command line: lazy dog." */static void __init print_kernel_cmdline(const char *cmdline){ size_t len;
/* Config option of 0 or anything longer than the max disables wrapping */ if (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN == 0 || IDEAL_CMDLINE_LEN >= COMMAND_LINE_SIZE - 1) { pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline); return; }
len = strlen(cmdline); while (len > IDEAL_CMDLINE_LEN) { const char *first_space; const char *prev_cutoff; const char *cutoff; int to_print; size_t used;
/* Find the last ' ' that wouldn't make the line too long */ prev_cutoff = NULL; cutoff = cmdline; while (true) { cutoff = strchr(cutoff + 1, ' '); if (!cutoff || cutoff - cmdline > IDEAL_CMDLINE_SPLIT_LEN) break; prev_cutoff = cutoff; } if (prev_cutoff) cutoff = prev_cutoff; else if (!cutoff) break;
/* Find the beginning and end of the string of spaces */ first_space = cutoff; while (first_space > cmdline && first_space[-1] == ' ') first_space--; to_print = first_space - cmdline; while (*cutoff == ' ') cutoff++; used = cutoff - cmdline;
/* If the whole string is used, break and do the final printout */ if (len == used) break;
if (to_print) pr_notice("%s%.*s%s\n", KERNEL_CMDLINE_PREFIX, to_print, cmdline, KERNEL_CMDLINE_CONTINUATION);
len -= used; cmdline += used; } if (len) pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline);}
asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protectorvoid start_kernel(void){ char *command_line; char *after_dashes;
set_task_stack_end_magic(&init_task); smp_setup_processor_id(); debug_objects_early_init(); init_vmlinux_build_id();
cgroup_init_early();
local_irq_disable(); early_boot_irqs_disabled = true;
/* * Interrupts are still disabled. Do necessary setups, then * enable them. */ boot_cpu_init(); page_address_init(); pr_notice("%s", linux_banner); setup_arch(&command_line); mm_core_init_early(); /* Static keys and static calls are needed by LSMs */ jump_label_init(); static_call_init(); early_security_init(); setup_boot_config(); setup_command_line(command_line); setup_nr_cpu_ids(); setup_per_cpu_areas(); smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ early_numa_node_init(); boot_cpu_hotplug_init();
print_kernel_cmdline(saved_command_line); /* parameters may set static keys */ parse_early_param(); after_dashes = parse_args("Booting kernel", static_command_line, __start___param, __stop___param - __start___param, -1, -1, NULL, &unknown_bootoption); print_unknown_bootoptions(); if (!IS_ERR_OR_NULL(after_dashes)) parse_args("Setting init args", after_dashes, NULL, 0, -1, -1, NULL, set_init_arg); if (extra_init_args) parse_args("Setting extra init args", extra_init_args, NULL, 0, -1, -1, NULL, set_init_arg);
/* Architectural and non-timekeeping rng init, before allocator init */ random_init_early(command_line);
/* * These use large bootmem allocations and must precede * initalization of page allocator */ setup_log_buf(0); vfs_caches_init_early(); sort_main_extable(); trap_init(); mm_core_init(); maple_tree_init(); poking_init(); ftrace_init();
/* trace_printk can be enabled here */ early_trace_init();
/* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init() * time - but meanwhile we still have a functioning scheduler. */ sched_init();
if (WARN(!irqs_disabled(), "Interrupts were enabled *very* early, fixing it\n")) local_irq_disable(); radix_tree_init();
/* * Set up housekeeping before setting up workqueues to allow the unbound * workqueue to take non-housekeeping into account. */ housekeeping_init();
/* * Allow workqueue creation and work item queueing/cancelling * early. Work item execution depends on kthreads and starts after * workqueue_init(). */ workqueue_init_early();
rcu_init(); kvfree_rcu_init();
/* Trace events are available after this */ trace_init();
if (initcall_debug) initcall_debug_enable();
context_tracking_init(); /* init some links before init_ISA_irqs() */ early_irq_init(); init_IRQ(); tick_init(); rcu_init_nohz(); timers_init(); srcu_init(); hrtimers_init(); softirq_init(); vdso_setup_data_pages(); timekeeping_init(); time_init();
/* This must be after timekeeping is initialized */ random_init();
/* These make use of the fully initialized rng */ kfence_init(); boot_init_stack_canary();
perf_event_init(); profile_init(); call_function_init(); WARN(!irqs_disabled(), "Interrupts were enabled early\n");
early_boot_irqs_disabled = false; local_irq_enable();
kmem_cache_init_late();
/* * HACK ALERT! This is early. We're enabling the console before * we've done PCI setups etc, and console_init() must be aware of * this. But we do want output early, in case something goes wrong. */ console_init(); if (panic_later) panic("Too many boot %s vars at `%s'", panic_later, panic_param);
lockdep_init();
/* * Need to run this when irqs are enabled, because it wants * to self-test [hard/soft]-irqs on/off lock inversion bugs * too: */ locking_selftest();
#ifdef CONFIG_BLK_DEV_INITRD if (initrd_start && !initrd_below_start_ok && page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) { pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n", page_to_pfn(virt_to_page((void *)initrd_start)), min_low_pfn); initrd_start = 0; }#endif setup_per_cpu_pageset(); numa_policy_init(); acpi_early_init(); if (late_time_init) late_time_init(); sched_clock_init(); calibrate_delay();
arch_cpu_finalize_init();
pid_idr_init(); anon_vma_init(); thread_stack_cache_init(); cred_init(); fork_init(); proc_caches_init(); uts_ns_init(); time_ns_init(); key_init(); security_init(); dbg_late_init(); net_ns_init(); vfs_caches_init(); pagecache_init(); signals_init(); seq_file_init(); proc_root_init(); nsfs_init(); pidfs_init(); cpuset_init(); mem_cgroup_init(); cgroup_init(); taskstats_init_early(); delayacct_init();
acpi_subsystem_init(); arch_post_acpi_subsys_init(); kcsan_init();
/* Do the rest non-__init'ed, we're now alive */ rest_init();
/* * Avoid stack canaries in callers of boot_init_stack_canary for gcc-10 * and older. */#if !__has_attribute(__no_stack_protector__) prevent_tail_call_optimization();#endif}
/* Call all constructor functions linked into the kernel. */static void __init do_ctors(void){/* * For UML, the constructors have already been called by the * normal setup code as it's just a normal ELF binary, so we * cannot do it again - but we do need CONFIG_CONSTRUCTORS * even on UML for modules. */#if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML) ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
for (; fn < (ctor_fn_t *) __ctors_end; fn++) (*fn)();#endif}
#ifdef CONFIG_KALLSYMSstruct blacklist_entry { struct list_head next; char *buf;};
static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
static int __init initcall_blacklist(char *str){ char *str_entry; struct blacklist_entry *entry;
/* str argument is a comma-separated list of functions */ do { str_entry = strsep(&str, ","); if (str_entry) { pr_debug("blacklisting initcall %s\n", str_entry); entry = memblock_alloc_or_panic(sizeof(*entry), SMP_CACHE_BYTES); entry->buf = memblock_alloc_or_panic(strlen(str_entry) + 1, SMP_CACHE_BYTES); strcpy(entry->buf, str_entry); list_add(&entry->next, &blacklisted_initcalls); } } while (str_entry);
return 1;}
static bool __init_or_module initcall_blacklisted(initcall_t fn){ struct blacklist_entry *entry; char fn_name[KSYM_SYMBOL_LEN]; unsigned long addr;
if (list_empty(&blacklisted_initcalls)) return false;
addr = (unsigned long) dereference_function_descriptor(fn); sprint_symbol_no_offset(fn_name, addr);
/* * fn will be "function_name [module_name]" where [module_name] is not * displayed for built-in init functions. Strip off the [module_name]. */ strreplace(fn_name, ' ', '\0');
list_for_each_entry(entry, &blacklisted_initcalls, next) { if (!strcmp(fn_name, entry->buf)) { pr_debug("initcall %s blacklisted\n", fn_name); return true; } }
return false;}#elsestatic int __init initcall_blacklist(char *str){ pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n"); return 0;}
static bool __init_or_module initcall_blacklisted(initcall_t fn){ return false;}#endif__setup("initcall_blacklist=", initcall_blacklist);
static __init_or_module voidtrace_initcall_start_cb(void *data, initcall_t fn){ ktime_t *calltime = data;
printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current)); *calltime = ktime_get();}
static __init_or_module voidtrace_initcall_finish_cb(void *data, initcall_t fn, int ret){ ktime_t rettime, *calltime = data;
rettime = ktime_get(); printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n", fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));}
static __init_or_module voidtrace_initcall_level_cb(void *data, const char *level){ printk(KERN_DEBUG "entering initcall level: %s\n", level);}
static ktime_t initcall_calltime;
#ifdef TRACEPOINTS_ENABLEDstatic void __init initcall_debug_enable(void){ int ret;
ret = register_trace_initcall_start(trace_initcall_start_cb, &initcall_calltime); ret |= register_trace_initcall_finish(trace_initcall_finish_cb, &initcall_calltime); ret |= register_trace_initcall_level(trace_initcall_level_cb, NULL); WARN(ret, "Failed to register initcall tracepoints\n");}# define do_trace_initcall_start trace_initcall_start# define do_trace_initcall_finish trace_initcall_finish# define do_trace_initcall_level trace_initcall_level#elsestatic inline void do_trace_initcall_start(initcall_t fn){ if (!initcall_debug) return; trace_initcall_start_cb(&initcall_calltime, fn);}static inline void do_trace_initcall_finish(initcall_t fn, int ret){ if (!initcall_debug) return; trace_initcall_finish_cb(&initcall_calltime, fn, ret);}static inline void do_trace_initcall_level(const char *level){ if (!initcall_debug) return; trace_initcall_level_cb(NULL, level);}#endif /* !TRACEPOINTS_ENABLED */
int __init_or_module do_one_initcall(initcall_t fn){ int count = preempt_count(); char msgbuf[64]; int ret;
if (initcall_blacklisted(fn)) return -EPERM;
do_trace_initcall_start(fn); ret = fn(); do_trace_initcall_finish(fn, ret);
msgbuf[0] = 0;
if (preempt_count() != count) { sprintf(msgbuf, "preemption imbalance "); preempt_count_set(count); } if (irqs_disabled()) { strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf)); local_irq_enable(); } WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
add_latent_entropy(); return ret;}
static initcall_entry_t *initcall_levels[] __initdata = { __initcall0_start, __initcall1_start, __initcall2_start, __initcall3_start, __initcall4_start, __initcall5_start, __initcall6_start, __initcall7_start, __initcall_end,};
/* Keep these in sync with initcalls in include/linux/init.h */static const char *initcall_level_names[] __initdata = { "pure", "core", "postcore", "arch", "subsys", "fs", "device", "late",};
static int __init ignore_unknown_bootoption(char *param, char *val, const char *unused, void *arg){ return 0;}
static void __init do_initcall_level(int level, char *command_line){ initcall_entry_t *fn;
parse_args(initcall_level_names[level], command_line, __start___param, __stop___param - __start___param, level, level, NULL, ignore_unknown_bootoption);
do_trace_initcall_level(initcall_level_names[level]); for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++) do_one_initcall(initcall_from_entry(fn));}
static void __init do_initcalls(void){ int level; size_t len = saved_command_line_len + 1; char *command_line;
command_line = kzalloc(len, GFP_KERNEL); if (!command_line) panic("%s: Failed to allocate %zu bytes\n", __func__, len);
for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) { /* Parser modifies command_line, restore it each time */ strcpy(command_line, saved_command_line); do_initcall_level(level, command_line); }
kfree(command_line);}
/* * Ok, the machine is now initialized. None of the devices * have been touched yet, but the CPU subsystem is up and * running, and memory and process management works. * * Now we can finally start doing some real work.. */static void __init do_basic_setup(void){ cpuset_init_smp(); ksysfs_init(); driver_init(); init_irq_proc(); do_ctors(); do_initcalls();}
static void __init do_pre_smp_initcalls(void){ initcall_entry_t *fn;
do_trace_initcall_level("early"); for (fn = __initcall_start; fn < __initcall0_start; fn++) do_one_initcall(initcall_from_entry(fn));}
static int run_init_process(const char *init_filename){ const char *const *p;
argv_init[0] = init_filename; pr_info("Run %s as init process\n", init_filename); pr_debug(" with arguments:\n"); for (p = argv_init; *p; p++) pr_debug(" %s\n", *p); pr_debug(" with environment:\n"); for (p = envp_init; *p; p++) pr_debug(" %s\n", *p); return kernel_execve(init_filename, argv_init, envp_init);}
static int try_to_run_init_process(const char *init_filename){ int ret;
ret = run_init_process(init_filename);
if (ret && ret != -ENOENT) { pr_err("Starting init: %s exists but couldn't execute it (error %d)\n", init_filename, ret); }
return ret;}
static noinline void __init kernel_init_freeable(void);
#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)bool rodata_enabled __ro_after_init = true;
#ifndef arch_parse_debug_rodatastatic inline bool arch_parse_debug_rodata(char *str) { return false; }#endif
static int __init set_debug_rodata(char *str){ if (arch_parse_debug_rodata(str)) return 0;
if (str && !strcmp(str, "on")) rodata_enabled = true; else if (str && !strcmp(str, "off")) rodata_enabled = false; else pr_warn("Invalid option string for rodata: '%s'\n", str); return 0;}early_param("rodata", set_debug_rodata);#endif
static void mark_readonly(void){ if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled) { /* * load_module() results in W+X mappings, which are cleaned * up with init_free_wq. Let's make sure that queued work is * flushed so that we don't hit false positives looking for * insecure pages which are W+X. */ flush_module_init_free_work(); jump_label_init_ro(); mark_rodata_ro(); debug_checkwx(); rodata_test(); } else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) { pr_info("Kernel memory protection disabled.\n"); } else if (IS_ENABLED(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)) { pr_warn("Kernel memory protection not selected by kernel config.\n"); } else { pr_warn("This architecture does not have kernel memory protection.\n"); }}
void __weak free_initmem(void){ free_initmem_default(POISON_FREE_INITMEM);}
static int __ref kernel_init(void *unused){ int ret;
/* * Wait until kthreadd is all set-up. */ wait_for_completion(&kthreadd_done);
kernel_init_freeable(); /* need to finish all async __init code before freeing the memory */ async_synchronize_full();
system_state = SYSTEM_FREEING_INITMEM; kprobe_free_init_mem(); ftrace_free_init_mem(); kgdb_free_init_mem(); exit_boot_config(); free_initmem(); mark_readonly();
/* * Kernel mappings are now finalized - update the userspace page-table * to finalize PTI. */ pti_finalize();
system_state = SYSTEM_RUNNING; numa_default_policy();
rcu_end_inkernel_boot();
do_sysctl_args();
if (ramdisk_execute_command) { ret = run_init_process(ramdisk_execute_command); if (!ret) return 0; pr_err("Failed to execute %s (error %d)\n", ramdisk_execute_command, ret); }
/* * We try each of these until one succeeds. * * The Bourne shell can be used instead of init if we are * trying to recover a really broken machine. */ if (execute_command) { ret = run_init_process(execute_command); if (!ret) return 0; panic("Requested init %s failed (error %d).", execute_command, ret); }
if (CONFIG_DEFAULT_INIT[0] != '\0') { ret = run_init_process(CONFIG_DEFAULT_INIT); if (ret) pr_err("Default init %s failed (error %d)\n", CONFIG_DEFAULT_INIT, ret); else return 0; }
if (!try_to_run_init_process("/sbin/init") || !try_to_run_init_process("/etc/init") || !try_to_run_init_process("/bin/init") || !try_to_run_init_process("/bin/sh")) return 0;
panic("No working init found. Try passing init= option to kernel. " "See Linux Documentation/admin-guide/init.rst for guidance.");}
/* Open /dev/console, for stdin/stdout/stderr, this should never fail */void __init console_on_rootfs(void){ struct file *file = filp_open("/dev/console", O_RDWR, 0);
if (IS_ERR(file)) { pr_err("Warning: unable to open an initial console.\n"); return; } init_dup(file); init_dup(file); init_dup(file); fput(file);}
static noinline void __init kernel_init_freeable(void){ /* Now the scheduler is fully set up and can do blocking allocations */ gfp_allowed_mask = __GFP_BITS_MASK;
/* * init can allocate pages on any node */ set_mems_allowed(node_states[N_MEMORY]);
cad_pid = get_pid(task_pid(current));
smp_prepare_cpus(setup_max_cpus);
workqueue_init();
init_mm_internals();
do_pre_smp_initcalls(); lockup_detector_init();
smp_init(); sched_init_smp();
workqueue_init_topology(); async_init(); padata_init(); page_alloc_init_late();
do_basic_setup();
kunit_run_all_tests();
wait_for_initramfs(); console_on_rootfs();
/* * check if there is an early userspace init. If yes, let it do all * the work */ int ramdisk_command_access; ramdisk_command_access = init_eaccess(ramdisk_execute_command); if (ramdisk_command_access != 0) { if (ramdisk_execute_command_set) pr_warn("check access for rdinit=%s failed: %i, ignoring\n", ramdisk_execute_command, ramdisk_command_access); ramdisk_execute_command = NULL; prepare_namespace(); }
/* * Ok, we have completed the initial bootup, and * we're essentially up and running. Get rid of the * initmem segments and start the user-mode stuff.. * * rootfs is available now, try loading the public keys * and default modules */
integrity_load_keys();}