1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/gfp.h>
9#include <linux/list.h>
10#include <linux/mmzone.h>
11#include <linux/rbtree.h>
12#include <linux/prio_tree.h>
13#include <linux/debug_locks.h>
14#include <linux/mm_types.h>
15
16struct mempolicy;
17struct anon_vma;
18struct file_ra_state;
19struct user_struct;
20struct writeback_control;
21
22#ifndef CONFIG_DISCONTIGMEM
23extern unsigned long max_mapnr;
24#endif
25
26extern unsigned long num_physpages;
27extern void * high_memory;
28extern int page_cluster;
29
30#ifdef CONFIG_SYSCTL
31extern int sysctl_legacy_va_layout;
32#else
33#define sysctl_legacy_va_layout 0
34#endif
35
36extern unsigned long mmap_min_addr;
37
38#include <asm/page.h>
39#include <asm/pgtable.h>
40#include <asm/processor.h>
41
42#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
43
44
45#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
46
47
48
49
50
51
52
53
54
55
56extern struct kmem_cache *vm_area_cachep;
57
58
59
60
61
62
63struct vm_list_struct {
64 struct vm_list_struct *next;
65 struct vm_area_struct *vma;
66};
67
68#ifndef CONFIG_MMU
69extern struct rb_root nommu_vma_tree;
70extern struct rw_semaphore nommu_vma_sem;
71
72extern unsigned int kobjsize(const void *objp);
73#endif
74
75
76
77
78#define VM_READ 0x00000001
79#define VM_WRITE 0x00000002
80#define VM_EXEC 0x00000004
81#define VM_SHARED 0x00000008
82
83
84#define VM_MAYREAD 0x00000010
85#define VM_MAYWRITE 0x00000020
86#define VM_MAYEXEC 0x00000040
87#define VM_MAYSHARE 0x00000080
88
89#define VM_GROWSDOWN 0x00000100
90#define VM_GROWSUP 0x00000200
91#define VM_PFNMAP 0x00000400
92#define VM_DENYWRITE 0x00000800
93
94#define VM_EXECUTABLE 0x00001000
95#define VM_LOCKED 0x00002000
96#define VM_IO 0x00004000
97
98
99#define VM_SEQ_READ 0x00008000
100#define VM_RAND_READ 0x00010000
101
102#define VM_DONTCOPY 0x00020000
103#define VM_DONTEXPAND 0x00040000
104#define VM_RESERVED 0x00080000
105#define VM_ACCOUNT 0x00100000
106#define VM_NORESERVE 0x00200000
107#define VM_HUGETLB 0x00400000
108#define VM_NONLINEAR 0x00800000
109#define VM_MAPPED_COPY 0x01000000
110#define VM_INSERTPAGE 0x02000000
111#define VM_ALWAYSDUMP 0x04000000
112
113#define VM_CAN_NONLINEAR 0x08000000
114#define VM_MIXEDMAP 0x10000000
115#define VM_SAO 0x20000000
116
117#ifndef VM_STACK_DEFAULT_FLAGS
118#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
119#endif
120
121#ifdef CONFIG_STACK_GROWSUP
122#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
123#else
124#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
125#endif
126
127#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
128#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
129#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
130#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
131#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
132
133
134
135
136
137extern pgprot_t protection_map[16];
138
139#define FAULT_FLAG_WRITE 0x01
140#define FAULT_FLAG_NONLINEAR 0x02
141#define FAULT_FLAG_MKWRITE 0x04
142
143
144
145
146
147
148
149
150
151
152
153struct vm_fault {
154 unsigned int flags;
155 pgoff_t pgoff;
156 void __user *virtual_address;
157
158 struct page *page;
159
160
161
162
163};
164
165
166
167
168
169
170struct vm_operations_struct {
171 void (*open)(struct vm_area_struct * area);
172 void (*close)(struct vm_area_struct * area);
173 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
174
175
176
177 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
178
179
180
181
182 int (*access)(struct vm_area_struct *vma, unsigned long addr,
183 void *buf, int len, int write);
184#ifdef CONFIG_NUMA
185
186
187
188
189
190
191
192 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
193
194
195
196
197
198
199
200
201
202
203
204 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
205 unsigned long addr);
206 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
207 const nodemask_t *to, unsigned long flags);
208#endif
209};
210
211struct mmu_gather;
212struct inode;
213
214#define page_private(page) ((page)->private)
215#define set_page_private(page, v) ((page)->private = (v))
216
217
218
219
220
221#include <linux/page-flags.h>
222
223#ifdef CONFIG_DEBUG_VM
224#define VM_BUG_ON(cond) BUG_ON(cond)
225#else
226#define VM_BUG_ON(condition) do { } while(0)
227#endif
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245static inline int put_page_testzero(struct page *page)
246{
247 VM_BUG_ON(atomic_read(&page->_count) == 0);
248 return atomic_dec_and_test(&page->_count);
249}
250
251
252
253
254
255static inline int get_page_unless_zero(struct page *page)
256{
257 return atomic_inc_not_zero(&page->_count);
258}
259
260
261struct page *vmalloc_to_page(const void *addr);
262unsigned long vmalloc_to_pfn(const void *addr);
263
264
265
266
267
268
269
270static inline int is_vmalloc_addr(const void *x)
271{
272#ifdef CONFIG_MMU
273 unsigned long addr = (unsigned long)x;
274
275 return addr >= VMALLOC_START && addr < VMALLOC_END;
276#else
277 return 0;
278#endif
279}
280
281static inline struct page *compound_head(struct page *page)
282{
283 if (unlikely(PageTail(page)))
284 return page->first_page;
285 return page;
286}
287
288static inline int page_count(struct page *page)
289{
290 return atomic_read(&compound_head(page)->_count);
291}
292
293static inline void get_page(struct page *page)
294{
295 page = compound_head(page);
296 VM_BUG_ON(atomic_read(&page->_count) == 0);
297 atomic_inc(&page->_count);
298}
299
300static inline struct page *virt_to_head_page(const void *x)
301{
302 struct page *page = virt_to_page(x);
303 return compound_head(page);
304}
305
306
307
308
309
310static inline void init_page_count(struct page *page)
311{
312 atomic_set(&page->_count, 1);
313}
314
315void put_page(struct page *page);
316void put_pages_list(struct list_head *pages);
317
318void split_page(struct page *page, unsigned int order);
319
320
321
322
323
324
325typedef void compound_page_dtor(struct page *);
326
327static inline void set_compound_page_dtor(struct page *page,
328 compound_page_dtor *dtor)
329{
330 page[1].lru.next = (void *)dtor;
331}
332
333static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
334{
335 return (compound_page_dtor *)page[1].lru.next;
336}
337
338static inline int compound_order(struct page *page)
339{
340 if (!PageHead(page))
341 return 0;
342 return (unsigned long)page[1].lru.prev;
343}
344
345static inline void set_compound_order(struct page *page, unsigned long order)
346{
347 page[1].lru.prev = (void *)order;
348}
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
431#define SECTIONS_WIDTH SECTIONS_SHIFT
432#else
433#define SECTIONS_WIDTH 0
434#endif
435
436#define ZONES_WIDTH ZONES_SHIFT
437
438#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
439#define NODES_WIDTH NODES_SHIFT
440#else
441#ifdef CONFIG_SPARSEMEM_VMEMMAP
442#error "Vmemmap: No space for nodes field in page flags"
443#endif
444#define NODES_WIDTH 0
445#endif
446
447
448#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
449#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
450#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
451
452
453
454
455
456#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
457#define NODE_NOT_IN_PAGE_FLAGS
458#endif
459
460#ifndef PFN_SECTION_SHIFT
461#define PFN_SECTION_SHIFT 0
462#endif
463
464
465
466
467
468
469#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
470#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
471#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
472
473
474#ifdef NODE_NOT_IN_PAGEFLAGS
475#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
476#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
477 SECTIONS_PGOFF : ZONES_PGOFF)
478#else
479#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
480#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
481 NODES_PGOFF : ZONES_PGOFF)
482#endif
483
484#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
485
486#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
487#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
488#endif
489
490#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
491#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
492#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
493#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
494
495static inline enum zone_type page_zonenum(struct page *page)
496{
497 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
498}
499
500
501
502
503
504
505
506
507
508static inline int page_zone_id(struct page *page)
509{
510 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
511}
512
513static inline int zone_to_nid(struct zone *zone)
514{
515#ifdef CONFIG_NUMA
516 return zone->node;
517#else
518 return 0;
519#endif
520}
521
522#ifdef NODE_NOT_IN_PAGE_FLAGS
523extern int page_to_nid(struct page *page);
524#else
525static inline int page_to_nid(struct page *page)
526{
527 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
528}
529#endif
530
531static inline struct zone *page_zone(struct page *page)
532{
533 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
534}
535
536#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
537static inline unsigned long page_to_section(struct page *page)
538{
539 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
540}
541#endif
542
543static inline void set_page_zone(struct page *page, enum zone_type zone)
544{
545 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
546 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
547}
548
549static inline void set_page_node(struct page *page, unsigned long node)
550{
551 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
552 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
553}
554
555static inline void set_page_section(struct page *page, unsigned long section)
556{
557 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
558 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
559}
560
561static inline void set_page_links(struct page *page, enum zone_type zone,
562 unsigned long node, unsigned long pfn)
563{
564 set_page_zone(page, zone);
565 set_page_node(page, node);
566 set_page_section(page, pfn_to_section_nr(pfn));
567}
568
569
570
571
572
573static inline unsigned long round_hint_to_min(unsigned long hint)
574{
575 hint &= PAGE_MASK;
576 if (((void *)hint != NULL) &&
577 (hint < mmap_min_addr))
578 return PAGE_ALIGN(mmap_min_addr);
579 return hint;
580}
581
582
583
584
585#include <linux/vmstat.h>
586
587static __always_inline void *lowmem_page_address(struct page *page)
588{
589 return __va(page_to_pfn(page) << PAGE_SHIFT);
590}
591
592#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
593#define HASHED_PAGE_VIRTUAL
594#endif
595
596#if defined(WANT_PAGE_VIRTUAL)
597#define page_address(page) ((page)->virtual)
598#define set_page_address(page, address) \
599 do { \
600 (page)->virtual = (address); \
601 } while(0)
602#define page_address_init() do { } while(0)
603#endif
604
605#if defined(HASHED_PAGE_VIRTUAL)
606void *page_address(struct page *page);
607void set_page_address(struct page *page, void *virtual);
608void page_address_init(void);
609#endif
610
611#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
612#define page_address(page) lowmem_page_address(page)
613#define set_page_address(page, address) do { } while(0)
614#define page_address_init() do { } while(0)
615#endif
616
617
618
619
620
621
622
623
624
625
626#define PAGE_MAPPING_ANON 1
627
628extern struct address_space swapper_space;
629static inline struct address_space *page_mapping(struct page *page)
630{
631 struct address_space *mapping = page->mapping;
632
633 VM_BUG_ON(PageSlab(page));
634#ifdef CONFIG_SWAP
635 if (unlikely(PageSwapCache(page)))
636 mapping = &swapper_space;
637 else
638#endif
639 if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
640 mapping = NULL;
641 return mapping;
642}
643
644static inline int PageAnon(struct page *page)
645{
646 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
647}
648
649
650
651
652
653static inline pgoff_t page_index(struct page *page)
654{
655 if (unlikely(PageSwapCache(page)))
656 return page_private(page);
657 return page->index;
658}
659
660
661
662
663
664
665static inline void reset_page_mapcount(struct page *page)
666{
667 atomic_set(&(page)->_mapcount, -1);
668}
669
670static inline int page_mapcount(struct page *page)
671{
672 return atomic_read(&(page)->_mapcount) + 1;
673}
674
675
676
677
678static inline int page_mapped(struct page *page)
679{
680 return atomic_read(&(page)->_mapcount) >= 0;
681}
682
683
684
685
686
687
688
689#define VM_FAULT_MINOR 0
690
691#define VM_FAULT_OOM 0x0001
692#define VM_FAULT_SIGBUS 0x0002
693#define VM_FAULT_MAJOR 0x0004
694#define VM_FAULT_WRITE 0x0008
695
696#define VM_FAULT_NOPAGE 0x0100
697#define VM_FAULT_LOCKED 0x0200
698
699#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS)
700
701#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
702
703extern void show_free_areas(void);
704
705#ifdef CONFIG_SHMEM
706int shmem_lock(struct file *file, int lock, struct user_struct *user);
707#else
708static inline int shmem_lock(struct file *file, int lock,
709 struct user_struct *user)
710{
711 return 0;
712}
713#endif
714struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
715
716int shmem_zero_setup(struct vm_area_struct *);
717
718#ifndef CONFIG_MMU
719extern unsigned long shmem_get_unmapped_area(struct file *file,
720 unsigned long addr,
721 unsigned long len,
722 unsigned long pgoff,
723 unsigned long flags);
724#endif
725
726extern int can_do_mlock(void);
727extern int user_shm_lock(size_t, struct user_struct *);
728extern void user_shm_unlock(size_t, struct user_struct *);
729
730
731
732
733struct zap_details {
734 struct vm_area_struct *nonlinear_vma;
735 struct address_space *check_mapping;
736 pgoff_t first_index;
737 pgoff_t last_index;
738 spinlock_t *i_mmap_lock;
739 unsigned long truncate_count;
740};
741
742struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
743 pte_t pte);
744
745int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
746 unsigned long size);
747unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
748 unsigned long size, struct zap_details *);
749unsigned long unmap_vmas(struct mmu_gather **tlb,
750 struct vm_area_struct *start_vma, unsigned long start_addr,
751 unsigned long end_addr, unsigned long *nr_accounted,
752 struct zap_details *);
753
754
755
756
757
758
759
760
761
762
763
764struct mm_walk {
765 int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
766 int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
767 int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
768 int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
769 int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
770 struct mm_struct *mm;
771 void *private;
772};
773
774int walk_page_range(unsigned long addr, unsigned long end,
775 struct mm_walk *walk);
776void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
777 unsigned long end, unsigned long floor, unsigned long ceiling);
778int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
779 struct vm_area_struct *vma);
780void unmap_mapping_range(struct address_space *mapping,
781 loff_t const holebegin, loff_t const holelen, int even_cows);
782int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
783 void *buf, int len, int write);
784
785static inline void unmap_shared_mapping_range(struct address_space *mapping,
786 loff_t const holebegin, loff_t const holelen)
787{
788 unmap_mapping_range(mapping, holebegin, holelen, 0);
789}
790
791extern int vmtruncate(struct inode * inode, loff_t offset);
792extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end);
793
794#ifdef CONFIG_MMU
795extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
796 unsigned long address, int write_access);
797#else
798static inline int handle_mm_fault(struct mm_struct *mm,
799 struct vm_area_struct *vma, unsigned long address,
800 int write_access)
801{
802
803 BUG();
804 return VM_FAULT_SIGBUS;
805}
806#endif
807
808extern int make_pages_present(unsigned long addr, unsigned long end);
809extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
810
811int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start,
812 int len, int write, int force, struct page **pages, struct vm_area_struct **vmas);
813
814extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
815extern void do_invalidatepage(struct page *page, unsigned long offset);
816
817int __set_page_dirty_nobuffers(struct page *page);
818int __set_page_dirty_no_writeback(struct page *page);
819int redirty_page_for_writepage(struct writeback_control *wbc,
820 struct page *page);
821int set_page_dirty(struct page *page);
822int set_page_dirty_lock(struct page *page);
823int clear_page_dirty_for_io(struct page *page);
824
825extern unsigned long move_page_tables(struct vm_area_struct *vma,
826 unsigned long old_addr, struct vm_area_struct *new_vma,
827 unsigned long new_addr, unsigned long len);
828extern unsigned long do_mremap(unsigned long addr,
829 unsigned long old_len, unsigned long new_len,
830 unsigned long flags, unsigned long new_addr);
831extern int mprotect_fixup(struct vm_area_struct *vma,
832 struct vm_area_struct **pprev, unsigned long start,
833 unsigned long end, unsigned long newflags);
834
835
836
837
838
839
840
841
842
843
844
845int get_user_pages_fast(unsigned long start, int nr_pages, int write,
846 struct page **pages);
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863struct shrinker {
864 int (*shrink)(int nr_to_scan, gfp_t gfp_mask);
865 int seeks;
866
867
868 struct list_head list;
869 long nr;
870};
871#define DEFAULT_SEEKS 2
872extern void register_shrinker(struct shrinker *);
873extern void unregister_shrinker(struct shrinker *);
874
875int vma_wants_writenotify(struct vm_area_struct *vma);
876
877extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
878
879#ifdef __PAGETABLE_PUD_FOLDED
880static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
881 unsigned long address)
882{
883 return 0;
884}
885#else
886int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
887#endif
888
889#ifdef __PAGETABLE_PMD_FOLDED
890static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
891 unsigned long address)
892{
893 return 0;
894}
895#else
896int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
897#endif
898
899int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
900int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
901
902
903
904
905
906#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
907static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
908{
909 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
910 NULL: pud_offset(pgd, address);
911}
912
913static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
914{
915 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
916 NULL: pmd_offset(pud, address);
917}
918#endif
919
920#if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS
921
922
923
924
925
926
927#define __pte_lockptr(page) &((page)->ptl)
928#define pte_lock_init(_page) do { \
929 spin_lock_init(__pte_lockptr(_page)); \
930} while (0)
931#define pte_lock_deinit(page) ((page)->mapping = NULL)
932#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
933#else
934
935
936
937#define pte_lock_init(page) do {} while (0)
938#define pte_lock_deinit(page) do {} while (0)
939#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
940#endif
941
942static inline void pgtable_page_ctor(struct page *page)
943{
944 pte_lock_init(page);
945 inc_zone_page_state(page, NR_PAGETABLE);
946}
947
948static inline void pgtable_page_dtor(struct page *page)
949{
950 pte_lock_deinit(page);
951 dec_zone_page_state(page, NR_PAGETABLE);
952}
953
954#define pte_offset_map_lock(mm, pmd, address, ptlp) \
955({ \
956 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
957 pte_t *__pte = pte_offset_map(pmd, address); \
958 *(ptlp) = __ptl; \
959 spin_lock(__ptl); \
960 __pte; \
961})
962
963#define pte_unmap_unlock(pte, ptl) do { \
964 spin_unlock(ptl); \
965 pte_unmap(pte); \
966} while (0)
967
968#define pte_alloc_map(mm, pmd, address) \
969 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
970 NULL: pte_offset_map(pmd, address))
971
972#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
973 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
974 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
975
976#define pte_alloc_kernel(pmd, address) \
977 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
978 NULL: pte_offset_kernel(pmd, address))
979
980extern void free_area_init(unsigned long * zones_size);
981extern void free_area_init_node(int nid, unsigned long * zones_size,
982 unsigned long zone_start_pfn, unsigned long *zholes_size);
983#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1012extern void add_active_range(unsigned int nid, unsigned long start_pfn,
1013 unsigned long end_pfn);
1014extern void remove_active_range(unsigned int nid, unsigned long start_pfn,
1015 unsigned long end_pfn);
1016extern void push_node_boundaries(unsigned int nid, unsigned long start_pfn,
1017 unsigned long end_pfn);
1018extern void remove_all_active_ranges(void);
1019extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1020 unsigned long end_pfn);
1021extern void get_pfn_range_for_nid(unsigned int nid,
1022 unsigned long *start_pfn, unsigned long *end_pfn);
1023extern unsigned long find_min_pfn_with_active_regions(void);
1024extern void free_bootmem_with_active_regions(int nid,
1025 unsigned long max_low_pfn);
1026typedef int (*work_fn_t)(unsigned long, unsigned long, void *);
1027extern void work_with_active_regions(int nid, work_fn_t work_fn, void *data);
1028extern void sparse_memory_present_with_active_regions(int nid);
1029#endif
1030
1031#if !defined(CONFIG_ARCH_POPULATES_NODE_MAP) && \
1032 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1033static inline int __early_pfn_to_nid(unsigned long pfn)
1034{
1035 return 0;
1036}
1037#else
1038
1039extern int __meminit early_pfn_to_nid(unsigned long pfn);
1040#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1041
1042extern int __meminit __early_pfn_to_nid(unsigned long pfn);
1043#endif
1044#endif
1045
1046extern void set_dma_reserve(unsigned long new_dma_reserve);
1047extern void memmap_init_zone(unsigned long, int, unsigned long,
1048 unsigned long, enum memmap_context);
1049extern void setup_per_zone_pages_min(void);
1050extern void mem_init(void);
1051extern void show_mem(void);
1052extern void si_meminfo(struct sysinfo * val);
1053extern void si_meminfo_node(struct sysinfo *val, int nid);
1054extern int after_bootmem;
1055
1056#ifdef CONFIG_NUMA
1057extern void setup_per_cpu_pageset(void);
1058#else
1059static inline void setup_per_cpu_pageset(void) {}
1060#endif
1061
1062
1063void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
1064void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
1065void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *);
1066struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma,
1067 struct prio_tree_iter *iter);
1068
1069#define vma_prio_tree_foreach(vma, iter, root, begin, end) \
1070 for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \
1071 (vma = vma_prio_tree_next(vma, iter)); )
1072
1073static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1074 struct list_head *list)
1075{
1076 vma->shared.vm_set.parent = NULL;
1077 list_add_tail(&vma->shared.vm_set.list, list);
1078}
1079
1080
1081extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1082extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
1083 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1084extern struct vm_area_struct *vma_merge(struct mm_struct *,
1085 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1086 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1087 struct mempolicy *);
1088extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1089extern int split_vma(struct mm_struct *,
1090 struct vm_area_struct *, unsigned long addr, int new_below);
1091extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1092extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1093 struct rb_node **, struct rb_node *);
1094extern void unlink_file_vma(struct vm_area_struct *);
1095extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1096 unsigned long addr, unsigned long len, pgoff_t pgoff);
1097extern void exit_mmap(struct mm_struct *);
1098
1099extern int mm_take_all_locks(struct mm_struct *mm);
1100extern void mm_drop_all_locks(struct mm_struct *mm);
1101
1102#ifdef CONFIG_PROC_FS
1103
1104extern void added_exe_file_vma(struct mm_struct *mm);
1105extern void removed_exe_file_vma(struct mm_struct *mm);
1106#else
1107static inline void added_exe_file_vma(struct mm_struct *mm)
1108{}
1109
1110static inline void removed_exe_file_vma(struct mm_struct *mm)
1111{}
1112#endif
1113
1114extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1115extern int install_special_mapping(struct mm_struct *mm,
1116 unsigned long addr, unsigned long len,
1117 unsigned long flags, struct page **pages);
1118
1119extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1120
1121extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1122 unsigned long len, unsigned long prot,
1123 unsigned long flag, unsigned long pgoff);
1124extern unsigned long mmap_region(struct file *file, unsigned long addr,
1125 unsigned long len, unsigned long flags,
1126 unsigned int vm_flags, unsigned long pgoff,
1127 int accountable);
1128
1129static inline unsigned long do_mmap(struct file *file, unsigned long addr,
1130 unsigned long len, unsigned long prot,
1131 unsigned long flag, unsigned long offset)
1132{
1133 unsigned long ret = -EINVAL;
1134 if ((offset + PAGE_ALIGN(len)) < offset)
1135 goto out;
1136 if (!(offset & ~PAGE_MASK))
1137 ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1138out:
1139 return ret;
1140}
1141
1142extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1143
1144extern unsigned long do_brk(unsigned long, unsigned long);
1145
1146
1147extern unsigned long page_unuse(struct page *);
1148extern void truncate_inode_pages(struct address_space *, loff_t);
1149extern void truncate_inode_pages_range(struct address_space *,
1150 loff_t lstart, loff_t lend);
1151
1152
1153extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1154
1155
1156int write_one_page(struct page *page, int wait);
1157
1158
1159#define VM_MAX_READAHEAD 128
1160#define VM_MIN_READAHEAD 16
1161
1162int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
1163 pgoff_t offset, unsigned long nr_to_read);
1164int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1165 pgoff_t offset, unsigned long nr_to_read);
1166
1167void page_cache_sync_readahead(struct address_space *mapping,
1168 struct file_ra_state *ra,
1169 struct file *filp,
1170 pgoff_t offset,
1171 unsigned long size);
1172
1173void page_cache_async_readahead(struct address_space *mapping,
1174 struct file_ra_state *ra,
1175 struct file *filp,
1176 struct page *pg,
1177 pgoff_t offset,
1178 unsigned long size);
1179
1180unsigned long max_sane_readahead(unsigned long nr);
1181
1182
1183extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1184#ifdef CONFIG_IA64
1185extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1186#endif
1187extern int expand_stack_downwards(struct vm_area_struct *vma,
1188 unsigned long address);
1189
1190
1191extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1192extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1193 struct vm_area_struct **pprev);
1194
1195
1196
1197static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1198{
1199 struct vm_area_struct * vma = find_vma(mm,start_addr);
1200
1201 if (vma && end_addr <= vma->vm_start)
1202 vma = NULL;
1203 return vma;
1204}
1205
1206static inline unsigned long vma_pages(struct vm_area_struct *vma)
1207{
1208 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1209}
1210
1211pgprot_t vm_get_page_prot(unsigned long vm_flags);
1212struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
1213int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1214 unsigned long pfn, unsigned long size, pgprot_t);
1215int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
1216int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1217 unsigned long pfn);
1218int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1219 unsigned long pfn);
1220
1221struct page *follow_page(struct vm_area_struct *, unsigned long address,
1222 unsigned int foll_flags);
1223#define FOLL_WRITE 0x01
1224#define FOLL_TOUCH 0x02
1225#define FOLL_GET 0x04
1226#define FOLL_ANON 0x08
1227
1228typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1229 void *data);
1230extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1231 unsigned long size, pte_fn_t fn, void *data);
1232
1233#ifdef CONFIG_PROC_FS
1234void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
1235#else
1236static inline void vm_stat_account(struct mm_struct *mm,
1237 unsigned long flags, struct file *file, long pages)
1238{
1239}
1240#endif
1241
1242#ifdef CONFIG_DEBUG_PAGEALLOC
1243extern int debug_pagealloc_enabled;
1244
1245extern void kernel_map_pages(struct page *page, int numpages, int enable);
1246
1247static inline void enable_debug_pagealloc(void)
1248{
1249 debug_pagealloc_enabled = 1;
1250}
1251#ifdef CONFIG_HIBERNATION
1252extern bool kernel_page_present(struct page *page);
1253#endif
1254#else
1255static inline void
1256kernel_map_pages(struct page *page, int numpages, int enable) {}
1257static inline void enable_debug_pagealloc(void)
1258{
1259}
1260#ifdef CONFIG_HIBERNATION
1261static inline bool kernel_page_present(struct page *page) { return true; }
1262#endif
1263#endif
1264
1265extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk);
1266#ifdef __HAVE_ARCH_GATE_AREA
1267int in_gate_area_no_task(unsigned long addr);
1268int in_gate_area(struct task_struct *task, unsigned long addr);
1269#else
1270int in_gate_area_no_task(unsigned long addr);
1271#define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);})
1272#endif
1273
1274int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *,
1275 void __user *, size_t *, loff_t *);
1276unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
1277 unsigned long lru_pages);
1278
1279#ifndef CONFIG_MMU
1280#define randomize_va_space 0
1281#else
1282extern int randomize_va_space;
1283#endif
1284
1285const char * arch_vma_name(struct vm_area_struct *vma);
1286void print_vma_addr(char *prefix, unsigned long rip);
1287
1288struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
1289pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
1290pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
1291pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
1292pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
1293void *vmemmap_alloc_block(unsigned long size, int node);
1294void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
1295int vmemmap_populate_basepages(struct page *start_page,
1296 unsigned long pages, int node);
1297int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
1298void vmemmap_populate_print_last(void);
1299
1300#endif
1301#endif
1302