1
2
3
4
5
6
7
8
9
10
11
12#include <linux/types.h>
13#include <linux/module.h>
14#include <linux/proc_fs.h>
15#include <linux/kernel.h>
16#include <linux/syscalls.h>
17#include <linux/stackprotector.h>
18#include <linux/string.h>
19#include <linux/ctype.h>
20#include <linux/delay.h>
21#include <linux/ioport.h>
22#include <linux/init.h>
23#include <linux/initrd.h>
24#include <linux/bootmem.h>
25#include <linux/acpi.h>
26#include <linux/tty.h>
27#include <linux/percpu.h>
28#include <linux/kmod.h>
29#include <linux/vmalloc.h>
30#include <linux/kernel_stat.h>
31#include <linux/start_kernel.h>
32#include <linux/security.h>
33#include <linux/smp.h>
34#include <linux/profile.h>
35#include <linux/rcupdate.h>
36#include <linux/moduleparam.h>
37#include <linux/kallsyms.h>
38#include <linux/writeback.h>
39#include <linux/cpu.h>
40#include <linux/cpuset.h>
41#include <linux/cgroup.h>
42#include <linux/efi.h>
43#include <linux/tick.h>
44#include <linux/interrupt.h>
45#include <linux/taskstats_kern.h>
46#include <linux/delayacct.h>
47#include <linux/unistd.h>
48#include <linux/rmap.h>
49#include <linux/mempolicy.h>
50#include <linux/key.h>
51#include <linux/buffer_head.h>
52#include <linux/page_cgroup.h>
53#include <linux/debug_locks.h>
54#include <linux/debugobjects.h>
55#include <linux/lockdep.h>
56#include <linux/kmemleak.h>
57#include <linux/pid_namespace.h>
58#include <linux/device.h>
59#include <linux/kthread.h>
60#include <linux/sched.h>
61#include <linux/signal.h>
62#include <linux/idr.h>
63#include <linux/kgdb.h>
64#include <linux/ftrace.h>
65#include <linux/async.h>
66#include <linux/kmemcheck.h>
67#include <linux/sfi.h>
68#include <linux/shmem_fs.h>
69#include <linux/slab.h>
70#include <linux/perf_event.h>
71#include <linux/file.h>
72#include <linux/ptrace.h>
73#include <linux/blkdev.h>
74#include <linux/elevator.h>
75
76#include <asm/io.h>
77#include <asm/bugs.h>
78#include <asm/setup.h>
79#include <asm/sections.h>
80#include <asm/cacheflush.h>
81
82#ifdef CONFIG_X86_LOCAL_APIC
83#include <asm/smp.h>
84#endif
85
86static int kernel_init(void *);
87
88extern void init_IRQ(void);
89extern void fork_init(unsigned long);
90extern void mca_init(void);
91extern void sbus_init(void);
92extern void radix_tree_init(void);
93#ifndef CONFIG_DEBUG_RODATA
94static inline void mark_rodata_ro(void) { }
95#endif
96
97#ifdef CONFIG_TC
98extern void tc_init(void);
99#endif
100
101
102
103
104
105
106
107
108bool early_boot_irqs_disabled __read_mostly;
109
110enum system_states system_state __read_mostly;
111EXPORT_SYMBOL(system_state);
112
113
114
115
116#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
117#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
118
119extern void time_init(void);
120
121void (*__initdata late_time_init)(void);
122extern void softirq_init(void);
123
124
125char __initdata boot_command_line[COMMAND_LINE_SIZE];
126
127char *saved_command_line;
128
129static char *static_command_line;
130
131static char *execute_command;
132static char *ramdisk_execute_command;
133
134
135
136
137
138
139
140
141
142
143unsigned int reset_devices;
144EXPORT_SYMBOL(reset_devices);
145
146static int __init set_reset_devices(char *str)
147{
148 reset_devices = 1;
149 return 1;
150}
151
152__setup("reset_devices", set_reset_devices);
153
154static const char * argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
155const char * envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
156static const char *panic_later, *panic_param;
157
158extern const struct obs_kernel_param __setup_start[], __setup_end[];
159
160static int __init obsolete_checksetup(char *line)
161{
162 const struct obs_kernel_param *p;
163 int had_early_param = 0;
164
165 p = __setup_start;
166 do {
167 int n = strlen(p->str);
168 if (parameqn(line, p->str, n)) {
169 if (p->early) {
170
171
172
173
174 if (line[n] == '\0' || line[n] == '=')
175 had_early_param = 1;
176 } else if (!p->setup_func) {
177 printk(KERN_WARNING "Parameter %s is obsolete,"
178 " ignored\n", p->str);
179 return 1;
180 } else if (p->setup_func(line + n))
181 return 1;
182 }
183 p++;
184 } while (p < __setup_end);
185
186 return had_early_param;
187}
188
189
190
191
192
193unsigned long loops_per_jiffy = (1<<12);
194
195EXPORT_SYMBOL(loops_per_jiffy);
196
197static int __init debug_kernel(char *str)
198{
199 console_loglevel = 10;
200 return 0;
201}
202
203static int __init quiet_kernel(char *str)
204{
205 console_loglevel = 4;
206 return 0;
207}
208
209early_param("debug", debug_kernel);
210early_param("quiet", quiet_kernel);
211
212static int __init loglevel(char *str)
213{
214 int newlevel;
215
216
217
218
219
220
221 if (get_option(&str, &newlevel)) {
222 console_loglevel = newlevel;
223 return 0;
224 }
225
226 return -EINVAL;
227}
228
229early_param("loglevel", loglevel);
230
231
232static int __init repair_env_string(char *param, char *val, const char *unused)
233{
234 if (val) {
235
236 if (val == param+strlen(param)+1)
237 val[-1] = '=';
238 else if (val == param+strlen(param)+2) {
239 val[-2] = '=';
240 memmove(val-1, val, strlen(val)+1);
241 val--;
242 } else
243 BUG();
244 }
245 return 0;
246}
247
248
249
250
251
252static int __init unknown_bootoption(char *param, char *val, const char *unused)
253{
254 repair_env_string(param, val, unused);
255
256
257 if (obsolete_checksetup(param))
258 return 0;
259
260
261 if (strchr(param, '.') && (!val || strchr(param, '.') < val))
262 return 0;
263
264 if (panic_later)
265 return 0;
266
267 if (val) {
268
269 unsigned int i;
270 for (i = 0; envp_init[i]; i++) {
271 if (i == MAX_INIT_ENVS) {
272 panic_later = "Too many boot env vars at `%s'";
273 panic_param = param;
274 }
275 if (!strncmp(param, envp_init[i], val - param))
276 break;
277 }
278 envp_init[i] = param;
279 } else {
280
281 unsigned int i;
282 for (i = 0; argv_init[i]; i++) {
283 if (i == MAX_INIT_ARGS) {
284 panic_later = "Too many boot init vars at `%s'";
285 panic_param = param;
286 }
287 }
288 argv_init[i] = param;
289 }
290 return 0;
291}
292
293static int __init init_setup(char *str)
294{
295 unsigned int i;
296
297 execute_command = str;
298
299
300
301
302
303
304 for (i = 1; i < MAX_INIT_ARGS; i++)
305 argv_init[i] = NULL;
306 return 1;
307}
308__setup("init=", init_setup);
309
310static int __init rdinit_setup(char *str)
311{
312 unsigned int i;
313
314 ramdisk_execute_command = str;
315
316 for (i = 1; i < MAX_INIT_ARGS; i++)
317 argv_init[i] = NULL;
318 return 1;
319}
320__setup("rdinit=", rdinit_setup);
321
322#ifndef CONFIG_SMP
323static const unsigned int setup_max_cpus = NR_CPUS;
324#ifdef CONFIG_X86_LOCAL_APIC
325static void __init smp_init(void)
326{
327 APIC_init_uniprocessor();
328}
329#else
330#define smp_init() do { } while (0)
331#endif
332
333static inline void setup_nr_cpu_ids(void) { }
334static inline void smp_prepare_cpus(unsigned int maxcpus) { }
335#endif
336
337
338
339
340
341
342
343static void __init setup_command_line(char *command_line)
344{
345 saved_command_line = alloc_bootmem(strlen (boot_command_line)+1);
346 static_command_line = alloc_bootmem(strlen (command_line)+1);
347 strcpy (saved_command_line, boot_command_line);
348 strcpy (static_command_line, command_line);
349}
350
351
352
353
354
355
356
357
358
359
360static __initdata DECLARE_COMPLETION(kthreadd_done);
361
362static noinline void __init_refok rest_init(void)
363{
364 int pid;
365
366 rcu_scheduler_starting();
367
368
369
370
371
372 kernel_thread(kernel_init, NULL, CLONE_FS | CLONE_SIGHAND);
373 numa_default_policy();
374 pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES);
375 rcu_read_lock();
376 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
377 rcu_read_unlock();
378 complete(&kthreadd_done);
379
380
381
382
383
384 init_idle_bootup_task(current);
385 schedule_preempt_disabled();
386
387 cpu_idle();
388}
389
390
391static int __init do_early_param(char *param, char *val, const char *unused)
392{
393 const struct obs_kernel_param *p;
394
395 for (p = __setup_start; p < __setup_end; p++) {
396 if ((p->early && parameq(param, p->str)) ||
397 (strcmp(param, "console") == 0 &&
398 strcmp(p->str, "earlycon") == 0)
399 ) {
400 if (p->setup_func(val) != 0)
401 printk(KERN_WARNING
402 "Malformed early option '%s'\n", param);
403 }
404 }
405
406 return 0;
407}
408
409void __init parse_early_options(char *cmdline)
410{
411 parse_args("early options", cmdline, NULL, 0, 0, 0, do_early_param);
412}
413
414
415void __init parse_early_param(void)
416{
417 static __initdata int done = 0;
418 static __initdata char tmp_cmdline[COMMAND_LINE_SIZE];
419
420 if (done)
421 return;
422
423
424 strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
425 parse_early_options(tmp_cmdline);
426 done = 1;
427}
428
429
430
431
432
433static void __init boot_cpu_init(void)
434{
435 int cpu = smp_processor_id();
436
437 set_cpu_online(cpu, true);
438 set_cpu_active(cpu, true);
439 set_cpu_present(cpu, true);
440 set_cpu_possible(cpu, true);
441}
442
443void __init __weak smp_setup_processor_id(void)
444{
445}
446
447# if THREAD_SIZE >= PAGE_SIZE
448void __init __weak thread_info_cache_init(void)
449{
450}
451#endif
452
453
454
455
456static void __init mm_init(void)
457{
458
459
460
461
462 page_cgroup_init_flatmem();
463 mem_init();
464 kmem_cache_init();
465 percpu_init_late();
466 pgtable_cache_init();
467 vmalloc_init();
468}
469
470asmlinkage void __init start_kernel(void)
471{
472 char * command_line;
473 extern const struct kernel_param __start___param[], __stop___param[];
474
475
476
477
478
479 lockdep_init();
480 smp_setup_processor_id();
481 debug_objects_early_init();
482
483
484
485
486 boot_init_stack_canary();
487
488 cgroup_init_early();
489
490 local_irq_disable();
491 early_boot_irqs_disabled = true;
492
493
494
495
496
497 tick_init();
498 boot_cpu_init();
499 page_address_init();
500 printk(KERN_NOTICE "%s", linux_banner);
501 setup_arch(&command_line);
502 mm_init_owner(&init_mm, &init_task);
503 mm_init_cpumask(&init_mm);
504 setup_command_line(command_line);
505 setup_nr_cpu_ids();
506 setup_per_cpu_areas();
507 smp_prepare_boot_cpu();
508
509 build_all_zonelists(NULL, NULL);
510 page_alloc_init();
511
512 printk(KERN_NOTICE "Kernel command line: %s\n", boot_command_line);
513 parse_early_param();
514 parse_args("Booting kernel", static_command_line, __start___param,
515 __stop___param - __start___param,
516 -1, -1, &unknown_bootoption);
517
518 jump_label_init();
519
520
521
522
523
524 setup_log_buf(0);
525 pidhash_init();
526 vfs_caches_init_early();
527 sort_main_extable();
528 trap_init();
529 mm_init();
530
531
532
533
534
535
536 sched_init();
537
538
539
540
541 preempt_disable();
542 if (!irqs_disabled()) {
543 printk(KERN_WARNING "start_kernel(): bug: interrupts were "
544 "enabled *very* early, fixing it\n");
545 local_irq_disable();
546 }
547 idr_init_cache();
548 perf_event_init();
549 rcu_init();
550 radix_tree_init();
551
552 early_irq_init();
553 init_IRQ();
554 init_timers();
555 hrtimers_init();
556 softirq_init();
557 timekeeping_init();
558 time_init();
559 profile_init();
560 call_function_init();
561 if (!irqs_disabled())
562 printk(KERN_CRIT "start_kernel(): bug: interrupts were "
563 "enabled early\n");
564 early_boot_irqs_disabled = false;
565 local_irq_enable();
566
567 kmem_cache_init_late();
568
569
570
571
572
573
574 console_init();
575 if (panic_later)
576 panic(panic_later, panic_param);
577
578 lockdep_info();
579
580
581
582
583
584
585 locking_selftest();
586
587#ifdef CONFIG_BLK_DEV_INITRD
588 if (initrd_start && !initrd_below_start_ok &&
589 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
590 printk(KERN_CRIT "initrd overwritten (0x%08lx < 0x%08lx) - "
591 "disabling it.\n",
592 page_to_pfn(virt_to_page((void *)initrd_start)),
593 min_low_pfn);
594 initrd_start = 0;
595 }
596#endif
597 page_cgroup_init();
598 debug_objects_mem_init();
599 kmemleak_init();
600 setup_per_cpu_pageset();
601 numa_policy_init();
602 if (late_time_init)
603 late_time_init();
604 sched_clock_init();
605 calibrate_delay();
606 pidmap_init();
607 anon_vma_init();
608#ifdef CONFIG_X86
609 if (efi_enabled(EFI_RUNTIME_SERVICES))
610 efi_enter_virtual_mode();
611#endif
612 thread_info_cache_init();
613 cred_init();
614 fork_init(totalram_pages);
615 proc_caches_init();
616 buffer_init();
617 key_init();
618 security_init();
619 dbg_late_init();
620 vfs_caches_init(totalram_pages);
621 signals_init();
622
623 page_writeback_init();
624#ifdef CONFIG_PROC_FS
625 proc_root_init();
626#endif
627 cgroup_init();
628 cpuset_init();
629 taskstats_init_early();
630 delayacct_init();
631
632 check_bugs();
633
634 acpi_early_init();
635 sfi_init_late();
636
637 if (efi_enabled(EFI_RUNTIME_SERVICES)) {
638 efi_late_init();
639 efi_free_boot_services();
640 }
641
642 ftrace_init();
643
644
645 rest_init();
646}
647
648
649static void __init do_ctors(void)
650{
651#ifdef CONFIG_CONSTRUCTORS
652 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
653
654 for (; fn < (ctor_fn_t *) __ctors_end; fn++)
655 (*fn)();
656#endif
657}
658
659bool initcall_debug;
660core_param(initcall_debug, initcall_debug, bool, 0644);
661
662static char msgbuf[64];
663
664static int __init_or_module do_one_initcall_debug(initcall_t fn)
665{
666 ktime_t calltime, delta, rettime;
667 unsigned long long duration;
668 int ret;
669
670 printk(KERN_DEBUG "calling %pF @ %i\n", fn, task_pid_nr(current));
671 calltime = ktime_get();
672 ret = fn();
673 rettime = ktime_get();
674 delta = ktime_sub(rettime, calltime);
675 duration = (unsigned long long) ktime_to_ns(delta) >> 10;
676 printk(KERN_DEBUG "initcall %pF returned %d after %lld usecs\n", fn,
677 ret, duration);
678
679 return ret;
680}
681
682int __init_or_module do_one_initcall(initcall_t fn)
683{
684 int count = preempt_count();
685 int ret;
686
687 if (initcall_debug)
688 ret = do_one_initcall_debug(fn);
689 else
690 ret = fn();
691
692 msgbuf[0] = 0;
693
694 if (ret && ret != -ENODEV && initcall_debug)
695 sprintf(msgbuf, "error code %d ", ret);
696
697 if (preempt_count() != count) {
698 strlcat(msgbuf, "preemption imbalance ", sizeof(msgbuf));
699 preempt_count() = count;
700 }
701 if (irqs_disabled()) {
702 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
703 local_irq_enable();
704 }
705 if (msgbuf[0]) {
706 printk("initcall %pF returned with %s\n", fn, msgbuf);
707 }
708
709 return ret;
710}
711
712
713extern initcall_t __initcall_start[];
714extern initcall_t __initcall0_start[];
715extern initcall_t __initcall1_start[];
716extern initcall_t __initcall2_start[];
717extern initcall_t __initcall3_start[];
718extern initcall_t __initcall4_start[];
719extern initcall_t __initcall5_start[];
720extern initcall_t __initcall6_start[];
721extern initcall_t __initcall7_start[];
722extern initcall_t __initcall_end[];
723
724static initcall_t *initcall_levels[] __initdata = {
725 __initcall0_start,
726 __initcall1_start,
727 __initcall2_start,
728 __initcall3_start,
729 __initcall4_start,
730 __initcall5_start,
731 __initcall6_start,
732 __initcall7_start,
733 __initcall_end,
734};
735
736
737static char *initcall_level_names[] __initdata = {
738 "early",
739 "core",
740 "postcore",
741 "arch",
742 "subsys",
743 "fs",
744 "device",
745 "late",
746};
747
748static void __init do_initcall_level(int level)
749{
750 extern const struct kernel_param __start___param[], __stop___param[];
751 initcall_t *fn;
752
753 strcpy(static_command_line, saved_command_line);
754 parse_args(initcall_level_names[level],
755 static_command_line, __start___param,
756 __stop___param - __start___param,
757 level, level,
758 &repair_env_string);
759
760 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
761 do_one_initcall(*fn);
762}
763
764static void __init do_initcalls(void)
765{
766 int level;
767
768 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++)
769 do_initcall_level(level);
770}
771
772
773
774
775
776
777
778
779static void __init do_basic_setup(void)
780{
781 cpuset_init_smp();
782 usermodehelper_init();
783 shmem_init();
784 driver_init();
785 init_irq_proc();
786 do_ctors();
787 usermodehelper_enable();
788 do_initcalls();
789}
790
791static void __init do_pre_smp_initcalls(void)
792{
793 initcall_t *fn;
794
795 for (fn = __initcall_start; fn < __initcall0_start; fn++)
796 do_one_initcall(*fn);
797}
798
799
800
801
802
803
804
805void __init load_default_modules(void)
806{
807 load_default_elevator_module();
808}
809
810static int run_init_process(const char *init_filename)
811{
812 argv_init[0] = init_filename;
813 return do_execve(init_filename,
814 (const char __user *const __user *)argv_init,
815 (const char __user *const __user *)envp_init);
816}
817
818static noinline void __init kernel_init_freeable(void);
819
820static int __ref kernel_init(void *unused)
821{
822 kernel_init_freeable();
823
824 async_synchronize_full();
825 free_initmem();
826 mark_rodata_ro();
827 system_state = SYSTEM_RUNNING;
828 numa_default_policy();
829
830 flush_delayed_fput();
831
832 if (ramdisk_execute_command) {
833 if (!run_init_process(ramdisk_execute_command))
834 return 0;
835 printk(KERN_WARNING "Failed to execute %s\n",
836 ramdisk_execute_command);
837 }
838
839
840
841
842
843
844
845 if (execute_command) {
846 if (!run_init_process(execute_command))
847 return 0;
848 printk(KERN_WARNING "Failed to execute %s. Attempting "
849 "defaults...\n", execute_command);
850 }
851 if (!run_init_process("/sbin/init") ||
852 !run_init_process("/etc/init") ||
853 !run_init_process("/bin/init") ||
854 !run_init_process("/bin/sh"))
855 return 0;
856
857 panic("No init found. Try passing init= option to kernel. "
858 "See Linux Documentation/init.txt for guidance.");
859}
860
861static noinline void __init kernel_init_freeable(void)
862{
863
864
865
866 wait_for_completion(&kthreadd_done);
867
868
869 gfp_allowed_mask = __GFP_BITS_MASK;
870
871
872
873
874 set_mems_allowed(node_states[N_MEMORY]);
875
876
877
878 set_cpus_allowed_ptr(current, cpu_all_mask);
879
880 cad_pid = task_pid(current);
881
882 smp_prepare_cpus(setup_max_cpus);
883
884 do_pre_smp_initcalls();
885 lockup_detector_init();
886
887 smp_init();
888 sched_init_smp();
889
890 do_basic_setup();
891
892
893 if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
894 printk(KERN_WARNING "Warning: unable to open an initial console.\n");
895
896 (void) sys_dup(0);
897 (void) sys_dup(0);
898
899
900
901
902
903 if (!ramdisk_execute_command)
904 ramdisk_execute_command = "/init";
905
906 if (sys_access((const char __user *) ramdisk_execute_command, 0) != 0) {
907 ramdisk_execute_command = NULL;
908 prepare_namespace();
909 }
910
911
912
913
914
915
916
917
918 load_default_modules();
919}
920