1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/gfp.h>
9#include <linux/list.h>
10#include <linux/mmdebug.h>
11#include <linux/mmzone.h>
12#include <linux/rbtree.h>
13#include <linux/prio_tree.h>
14#include <linux/debug_locks.h>
15#include <linux/mm_types.h>
16
17struct mempolicy;
18struct anon_vma;
19struct file_ra_state;
20struct user_struct;
21struct writeback_control;
22
23#ifndef CONFIG_DISCONTIGMEM
24extern unsigned long max_mapnr;
25#endif
26
27extern unsigned long num_physpages;
28extern void * high_memory;
29extern int page_cluster;
30
31#ifdef CONFIG_SYSCTL
32extern int sysctl_legacy_va_layout;
33#else
34#define sysctl_legacy_va_layout 0
35#endif
36
37extern unsigned long mmap_min_addr;
38
39#include <asm/page.h>
40#include <asm/pgtable.h>
41#include <asm/processor.h>
42
43#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
44
45
46#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
47
48
49
50
51
52
53
54
55
56
57extern struct kmem_cache *vm_area_cachep;
58
59#ifndef CONFIG_MMU
60extern struct rb_root nommu_region_tree;
61extern struct rw_semaphore nommu_region_sem;
62
63extern unsigned int kobjsize(const void *objp);
64#endif
65
66
67
68
69#define VM_READ 0x00000001
70#define VM_WRITE 0x00000002
71#define VM_EXEC 0x00000004
72#define VM_SHARED 0x00000008
73
74
75#define VM_MAYREAD 0x00000010
76#define VM_MAYWRITE 0x00000020
77#define VM_MAYEXEC 0x00000040
78#define VM_MAYSHARE 0x00000080
79
80#define VM_GROWSDOWN 0x00000100
81#define VM_GROWSUP 0x00000200
82#define VM_PFNMAP 0x00000400
83#define VM_DENYWRITE 0x00000800
84
85#define VM_EXECUTABLE 0x00001000
86#define VM_LOCKED 0x00002000
87#define VM_IO 0x00004000
88
89
90#define VM_SEQ_READ 0x00008000
91#define VM_RAND_READ 0x00010000
92
93#define VM_DONTCOPY 0x00020000
94#define VM_DONTEXPAND 0x00040000
95#define VM_RESERVED 0x00080000
96#define VM_ACCOUNT 0x00100000
97#define VM_NORESERVE 0x00200000
98#define VM_HUGETLB 0x00400000
99#define VM_NONLINEAR 0x00800000
100#define VM_MAPPED_COPY 0x01000000
101#define VM_INSERTPAGE 0x02000000
102#define VM_ALWAYSDUMP 0x04000000
103
104#define VM_CAN_NONLINEAR 0x08000000
105#define VM_MIXEDMAP 0x10000000
106#define VM_SAO 0x20000000
107#define VM_PFN_AT_MMAP 0x40000000
108
109#ifndef VM_STACK_DEFAULT_FLAGS
110#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
111#endif
112
113#ifdef CONFIG_STACK_GROWSUP
114#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
115#else
116#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
117#endif
118
119#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
120#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
121#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
122#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
123#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
124
125
126
127
128#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_RESERVED | VM_PFNMAP)
129
130
131
132
133
134extern pgprot_t protection_map[16];
135
136#define FAULT_FLAG_WRITE 0x01
137#define FAULT_FLAG_NONLINEAR 0x02
138#define FAULT_FLAG_MKWRITE 0x04
139
140
141
142
143
144
145
146
147
148static inline int is_linear_pfn_mapping(struct vm_area_struct *vma)
149{
150 return (vma->vm_flags & VM_PFN_AT_MMAP);
151}
152
153static inline int is_pfn_mapping(struct vm_area_struct *vma)
154{
155 return (vma->vm_flags & VM_PFNMAP);
156}
157
158
159
160
161
162
163
164
165
166
167struct vm_fault {
168 unsigned int flags;
169 pgoff_t pgoff;
170 void __user *virtual_address;
171
172 struct page *page;
173
174
175
176
177};
178
179
180
181
182
183
184struct vm_operations_struct {
185 void (*open)(struct vm_area_struct * area);
186 void (*close)(struct vm_area_struct * area);
187 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
188
189
190
191 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
192
193
194
195
196 int (*access)(struct vm_area_struct *vma, unsigned long addr,
197 void *buf, int len, int write);
198#ifdef CONFIG_NUMA
199
200
201
202
203
204
205
206 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
207
208
209
210
211
212
213
214
215
216
217
218 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
219 unsigned long addr);
220 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
221 const nodemask_t *to, unsigned long flags);
222#endif
223};
224
225struct mmu_gather;
226struct inode;
227
228#define page_private(page) ((page)->private)
229#define set_page_private(page, v) ((page)->private = (v))
230
231
232
233
234
235#include <linux/page-flags.h>
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253static inline int put_page_testzero(struct page *page)
254{
255 VM_BUG_ON(atomic_read(&page->_count) == 0);
256 return atomic_dec_and_test(&page->_count);
257}
258
259
260
261
262
263static inline int get_page_unless_zero(struct page *page)
264{
265 return atomic_inc_not_zero(&page->_count);
266}
267
268
269struct page *vmalloc_to_page(const void *addr);
270unsigned long vmalloc_to_pfn(const void *addr);
271
272
273
274
275
276
277
278static inline int is_vmalloc_addr(const void *x)
279{
280#ifdef CONFIG_MMU
281 unsigned long addr = (unsigned long)x;
282
283 return addr >= VMALLOC_START && addr < VMALLOC_END;
284#else
285 return 0;
286#endif
287}
288
289static inline struct page *compound_head(struct page *page)
290{
291 if (unlikely(PageTail(page)))
292 return page->first_page;
293 return page;
294}
295
296static inline int page_count(struct page *page)
297{
298 return atomic_read(&compound_head(page)->_count);
299}
300
301static inline void get_page(struct page *page)
302{
303 page = compound_head(page);
304 VM_BUG_ON(atomic_read(&page->_count) == 0);
305 atomic_inc(&page->_count);
306}
307
308static inline struct page *virt_to_head_page(const void *x)
309{
310 struct page *page = virt_to_page(x);
311 return compound_head(page);
312}
313
314
315
316
317
318static inline void init_page_count(struct page *page)
319{
320 atomic_set(&page->_count, 1);
321}
322
323void put_page(struct page *page);
324void put_pages_list(struct list_head *pages);
325
326void split_page(struct page *page, unsigned int order);
327
328
329
330
331
332
333typedef void compound_page_dtor(struct page *);
334
335static inline void set_compound_page_dtor(struct page *page,
336 compound_page_dtor *dtor)
337{
338 page[1].lru.next = (void *)dtor;
339}
340
341static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
342{
343 return (compound_page_dtor *)page[1].lru.next;
344}
345
346static inline int compound_order(struct page *page)
347{
348 if (!PageHead(page))
349 return 0;
350 return (unsigned long)page[1].lru.prev;
351}
352
353static inline void set_compound_order(struct page *page, unsigned long order)
354{
355 page[1].lru.prev = (void *)order;
356}
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
439#define SECTIONS_WIDTH SECTIONS_SHIFT
440#else
441#define SECTIONS_WIDTH 0
442#endif
443
444#define ZONES_WIDTH ZONES_SHIFT
445
446#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
447#define NODES_WIDTH NODES_SHIFT
448#else
449#ifdef CONFIG_SPARSEMEM_VMEMMAP
450#error "Vmemmap: No space for nodes field in page flags"
451#endif
452#define NODES_WIDTH 0
453#endif
454
455
456#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
457#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
458#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
459
460
461
462
463
464#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
465#define NODE_NOT_IN_PAGE_FLAGS
466#endif
467
468#ifndef PFN_SECTION_SHIFT
469#define PFN_SECTION_SHIFT 0
470#endif
471
472
473
474
475
476
477#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
478#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
479#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
480
481
482#ifdef NODE_NOT_IN_PAGEFLAGS
483#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
484#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
485 SECTIONS_PGOFF : ZONES_PGOFF)
486#else
487#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
488#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
489 NODES_PGOFF : ZONES_PGOFF)
490#endif
491
492#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
493
494#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
495#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
496#endif
497
498#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
499#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
500#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
501#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
502
503static inline enum zone_type page_zonenum(struct page *page)
504{
505 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
506}
507
508
509
510
511
512
513
514
515
516static inline int page_zone_id(struct page *page)
517{
518 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
519}
520
521static inline int zone_to_nid(struct zone *zone)
522{
523#ifdef CONFIG_NUMA
524 return zone->node;
525#else
526 return 0;
527#endif
528}
529
530#ifdef NODE_NOT_IN_PAGE_FLAGS
531extern int page_to_nid(struct page *page);
532#else
533static inline int page_to_nid(struct page *page)
534{
535 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
536}
537#endif
538
539static inline struct zone *page_zone(struct page *page)
540{
541 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
542}
543
544#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
545static inline unsigned long page_to_section(struct page *page)
546{
547 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
548}
549#endif
550
551static inline void set_page_zone(struct page *page, enum zone_type zone)
552{
553 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
554 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
555}
556
557static inline void set_page_node(struct page *page, unsigned long node)
558{
559 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
560 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
561}
562
563static inline void set_page_section(struct page *page, unsigned long section)
564{
565 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
566 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
567}
568
569static inline void set_page_links(struct page *page, enum zone_type zone,
570 unsigned long node, unsigned long pfn)
571{
572 set_page_zone(page, zone);
573 set_page_node(page, node);
574 set_page_section(page, pfn_to_section_nr(pfn));
575}
576
577
578
579
580
581static inline unsigned long round_hint_to_min(unsigned long hint)
582{
583#ifdef CONFIG_SECURITY
584 hint &= PAGE_MASK;
585 if (((void *)hint != NULL) &&
586 (hint < mmap_min_addr))
587 return PAGE_ALIGN(mmap_min_addr);
588#endif
589 return hint;
590}
591
592
593
594
595#include <linux/vmstat.h>
596
597static __always_inline void *lowmem_page_address(struct page *page)
598{
599 return __va(page_to_pfn(page) << PAGE_SHIFT);
600}
601
602#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
603#define HASHED_PAGE_VIRTUAL
604#endif
605
606#if defined(WANT_PAGE_VIRTUAL)
607#define page_address(page) ((page)->virtual)
608#define set_page_address(page, address) \
609 do { \
610 (page)->virtual = (address); \
611 } while(0)
612#define page_address_init() do { } while(0)
613#endif
614
615#if defined(HASHED_PAGE_VIRTUAL)
616void *page_address(struct page *page);
617void set_page_address(struct page *page, void *virtual);
618void page_address_init(void);
619#endif
620
621#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
622#define page_address(page) lowmem_page_address(page)
623#define set_page_address(page, address) do { } while(0)
624#define page_address_init() do { } while(0)
625#endif
626
627
628
629
630
631
632
633
634
635
636#define PAGE_MAPPING_ANON 1
637
638extern struct address_space swapper_space;
639static inline struct address_space *page_mapping(struct page *page)
640{
641 struct address_space *mapping = page->mapping;
642
643 VM_BUG_ON(PageSlab(page));
644#ifdef CONFIG_SWAP
645 if (unlikely(PageSwapCache(page)))
646 mapping = &swapper_space;
647 else
648#endif
649 if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
650 mapping = NULL;
651 return mapping;
652}
653
654static inline int PageAnon(struct page *page)
655{
656 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
657}
658
659
660
661
662
663static inline pgoff_t page_index(struct page *page)
664{
665 if (unlikely(PageSwapCache(page)))
666 return page_private(page);
667 return page->index;
668}
669
670
671
672
673
674
675static inline void reset_page_mapcount(struct page *page)
676{
677 atomic_set(&(page)->_mapcount, -1);
678}
679
680static inline int page_mapcount(struct page *page)
681{
682 return atomic_read(&(page)->_mapcount) + 1;
683}
684
685
686
687
688static inline int page_mapped(struct page *page)
689{
690 return atomic_read(&(page)->_mapcount) >= 0;
691}
692
693
694
695
696
697
698
699#define VM_FAULT_MINOR 0
700
701#define VM_FAULT_OOM 0x0001
702#define VM_FAULT_SIGBUS 0x0002
703#define VM_FAULT_MAJOR 0x0004
704#define VM_FAULT_WRITE 0x0008
705
706#define VM_FAULT_NOPAGE 0x0100
707#define VM_FAULT_LOCKED 0x0200
708
709#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS)
710
711
712
713
714extern void pagefault_out_of_memory(void);
715
716#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
717
718extern void show_free_areas(void);
719
720#ifdef CONFIG_SHMEM
721extern int shmem_lock(struct file *file, int lock, struct user_struct *user);
722#else
723static inline int shmem_lock(struct file *file, int lock,
724 struct user_struct *user)
725{
726 return 0;
727}
728#endif
729struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
730
731int shmem_zero_setup(struct vm_area_struct *);
732
733#ifndef CONFIG_MMU
734extern unsigned long shmem_get_unmapped_area(struct file *file,
735 unsigned long addr,
736 unsigned long len,
737 unsigned long pgoff,
738 unsigned long flags);
739#endif
740
741extern int can_do_mlock(void);
742extern int user_shm_lock(size_t, struct user_struct *);
743extern void user_shm_unlock(size_t, struct user_struct *);
744
745
746
747
748struct zap_details {
749 struct vm_area_struct *nonlinear_vma;
750 struct address_space *check_mapping;
751 pgoff_t first_index;
752 pgoff_t last_index;
753 spinlock_t *i_mmap_lock;
754 unsigned long truncate_count;
755};
756
757struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
758 pte_t pte);
759
760int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
761 unsigned long size);
762unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
763 unsigned long size, struct zap_details *);
764unsigned long unmap_vmas(struct mmu_gather **tlb,
765 struct vm_area_struct *start_vma, unsigned long start_addr,
766 unsigned long end_addr, unsigned long *nr_accounted,
767 struct zap_details *);
768
769
770
771
772
773
774
775
776
777
778
779struct mm_walk {
780 int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
781 int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
782 int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
783 int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
784 int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
785 struct mm_struct *mm;
786 void *private;
787};
788
789int walk_page_range(unsigned long addr, unsigned long end,
790 struct mm_walk *walk);
791void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
792 unsigned long end, unsigned long floor, unsigned long ceiling);
793int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
794 struct vm_area_struct *vma);
795void unmap_mapping_range(struct address_space *mapping,
796 loff_t const holebegin, loff_t const holelen, int even_cows);
797int follow_phys(struct vm_area_struct *vma, unsigned long address,
798 unsigned int flags, unsigned long *prot, resource_size_t *phys);
799int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
800 void *buf, int len, int write);
801
802static inline void unmap_shared_mapping_range(struct address_space *mapping,
803 loff_t const holebegin, loff_t const holelen)
804{
805 unmap_mapping_range(mapping, holebegin, holelen, 0);
806}
807
808extern int vmtruncate(struct inode * inode, loff_t offset);
809extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end);
810
811#ifdef CONFIG_MMU
812extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
813 unsigned long address, int write_access);
814#else
815static inline int handle_mm_fault(struct mm_struct *mm,
816 struct vm_area_struct *vma, unsigned long address,
817 int write_access)
818{
819
820 BUG();
821 return VM_FAULT_SIGBUS;
822}
823#endif
824
825extern int make_pages_present(unsigned long addr, unsigned long end);
826extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
827
828int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start,
829 int len, int write, int force, struct page **pages, struct vm_area_struct **vmas);
830
831extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
832extern void do_invalidatepage(struct page *page, unsigned long offset);
833
834int __set_page_dirty_nobuffers(struct page *page);
835int __set_page_dirty_no_writeback(struct page *page);
836int redirty_page_for_writepage(struct writeback_control *wbc,
837 struct page *page);
838void account_page_dirtied(struct page *page, struct address_space *mapping);
839int set_page_dirty(struct page *page);
840int set_page_dirty_lock(struct page *page);
841int clear_page_dirty_for_io(struct page *page);
842
843extern unsigned long move_page_tables(struct vm_area_struct *vma,
844 unsigned long old_addr, struct vm_area_struct *new_vma,
845 unsigned long new_addr, unsigned long len);
846extern unsigned long do_mremap(unsigned long addr,
847 unsigned long old_len, unsigned long new_len,
848 unsigned long flags, unsigned long new_addr);
849extern int mprotect_fixup(struct vm_area_struct *vma,
850 struct vm_area_struct **pprev, unsigned long start,
851 unsigned long end, unsigned long newflags);
852
853
854
855
856
857
858
859
860
861
862
863int get_user_pages_fast(unsigned long start, int nr_pages, int write,
864 struct page **pages);
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881struct shrinker {
882 int (*shrink)(int nr_to_scan, gfp_t gfp_mask);
883 int seeks;
884
885
886 struct list_head list;
887 long nr;
888};
889#define DEFAULT_SEEKS 2
890extern void register_shrinker(struct shrinker *);
891extern void unregister_shrinker(struct shrinker *);
892
893int vma_wants_writenotify(struct vm_area_struct *vma);
894
895extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
896
897#ifdef __PAGETABLE_PUD_FOLDED
898static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
899 unsigned long address)
900{
901 return 0;
902}
903#else
904int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
905#endif
906
907#ifdef __PAGETABLE_PMD_FOLDED
908static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
909 unsigned long address)
910{
911 return 0;
912}
913#else
914int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
915#endif
916
917int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
918int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
919
920
921
922
923
924#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
925static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
926{
927 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
928 NULL: pud_offset(pgd, address);
929}
930
931static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
932{
933 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
934 NULL: pmd_offset(pud, address);
935}
936#endif
937
938#if USE_SPLIT_PTLOCKS
939
940
941
942
943
944
945#define __pte_lockptr(page) &((page)->ptl)
946#define pte_lock_init(_page) do { \
947 spin_lock_init(__pte_lockptr(_page)); \
948} while (0)
949#define pte_lock_deinit(page) ((page)->mapping = NULL)
950#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
951#else
952
953
954
955#define pte_lock_init(page) do {} while (0)
956#define pte_lock_deinit(page) do {} while (0)
957#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
958#endif
959
960static inline void pgtable_page_ctor(struct page *page)
961{
962 pte_lock_init(page);
963 inc_zone_page_state(page, NR_PAGETABLE);
964}
965
966static inline void pgtable_page_dtor(struct page *page)
967{
968 pte_lock_deinit(page);
969 dec_zone_page_state(page, NR_PAGETABLE);
970}
971
972#define pte_offset_map_lock(mm, pmd, address, ptlp) \
973({ \
974 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
975 pte_t *__pte = pte_offset_map(pmd, address); \
976 *(ptlp) = __ptl; \
977 spin_lock(__ptl); \
978 __pte; \
979})
980
981#define pte_unmap_unlock(pte, ptl) do { \
982 spin_unlock(ptl); \
983 pte_unmap(pte); \
984} while (0)
985
986#define pte_alloc_map(mm, pmd, address) \
987 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
988 NULL: pte_offset_map(pmd, address))
989
990#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
991 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
992 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
993
994#define pte_alloc_kernel(pmd, address) \
995 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
996 NULL: pte_offset_kernel(pmd, address))
997
998extern void free_area_init(unsigned long * zones_size);
999extern void free_area_init_node(int nid, unsigned long * zones_size,
1000 unsigned long zone_start_pfn, unsigned long *zholes_size);
1001#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1030extern void add_active_range(unsigned int nid, unsigned long start_pfn,
1031 unsigned long end_pfn);
1032extern void remove_active_range(unsigned int nid, unsigned long start_pfn,
1033 unsigned long end_pfn);
1034extern void push_node_boundaries(unsigned int nid, unsigned long start_pfn,
1035 unsigned long end_pfn);
1036extern void remove_all_active_ranges(void);
1037extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1038 unsigned long end_pfn);
1039extern void get_pfn_range_for_nid(unsigned int nid,
1040 unsigned long *start_pfn, unsigned long *end_pfn);
1041extern unsigned long find_min_pfn_with_active_regions(void);
1042extern void free_bootmem_with_active_regions(int nid,
1043 unsigned long max_low_pfn);
1044typedef int (*work_fn_t)(unsigned long, unsigned long, void *);
1045extern void work_with_active_regions(int nid, work_fn_t work_fn, void *data);
1046extern void sparse_memory_present_with_active_regions(int nid);
1047#endif
1048
1049#if !defined(CONFIG_ARCH_POPULATES_NODE_MAP) && \
1050 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1051static inline int __early_pfn_to_nid(unsigned long pfn)
1052{
1053 return 0;
1054}
1055#else
1056
1057extern int __meminit early_pfn_to_nid(unsigned long pfn);
1058#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1059
1060extern int __meminit __early_pfn_to_nid(unsigned long pfn);
1061#endif
1062#endif
1063
1064extern void set_dma_reserve(unsigned long new_dma_reserve);
1065extern void memmap_init_zone(unsigned long, int, unsigned long,
1066 unsigned long, enum memmap_context);
1067extern void setup_per_zone_pages_min(void);
1068extern void mem_init(void);
1069extern void __init mmap_init(void);
1070extern void show_mem(void);
1071extern void si_meminfo(struct sysinfo * val);
1072extern void si_meminfo_node(struct sysinfo *val, int nid);
1073extern int after_bootmem;
1074
1075#ifdef CONFIG_NUMA
1076extern void setup_per_cpu_pageset(void);
1077#else
1078static inline void setup_per_cpu_pageset(void) {}
1079#endif
1080
1081
1082extern atomic_long_t mmap_pages_allocated;
1083
1084
1085void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
1086void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
1087void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *);
1088struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma,
1089 struct prio_tree_iter *iter);
1090
1091#define vma_prio_tree_foreach(vma, iter, root, begin, end) \
1092 for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \
1093 (vma = vma_prio_tree_next(vma, iter)); )
1094
1095static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1096 struct list_head *list)
1097{
1098 vma->shared.vm_set.parent = NULL;
1099 list_add_tail(&vma->shared.vm_set.list, list);
1100}
1101
1102
1103extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1104extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
1105 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1106extern struct vm_area_struct *vma_merge(struct mm_struct *,
1107 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1108 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1109 struct mempolicy *);
1110extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1111extern int split_vma(struct mm_struct *,
1112 struct vm_area_struct *, unsigned long addr, int new_below);
1113extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1114extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1115 struct rb_node **, struct rb_node *);
1116extern void unlink_file_vma(struct vm_area_struct *);
1117extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1118 unsigned long addr, unsigned long len, pgoff_t pgoff);
1119extern void exit_mmap(struct mm_struct *);
1120
1121extern int mm_take_all_locks(struct mm_struct *mm);
1122extern void mm_drop_all_locks(struct mm_struct *mm);
1123
1124#ifdef CONFIG_PROC_FS
1125
1126extern void added_exe_file_vma(struct mm_struct *mm);
1127extern void removed_exe_file_vma(struct mm_struct *mm);
1128#else
1129static inline void added_exe_file_vma(struct mm_struct *mm)
1130{}
1131
1132static inline void removed_exe_file_vma(struct mm_struct *mm)
1133{}
1134#endif
1135
1136extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1137extern int install_special_mapping(struct mm_struct *mm,
1138 unsigned long addr, unsigned long len,
1139 unsigned long flags, struct page **pages);
1140
1141extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1142
1143extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1144 unsigned long len, unsigned long prot,
1145 unsigned long flag, unsigned long pgoff);
1146extern unsigned long mmap_region(struct file *file, unsigned long addr,
1147 unsigned long len, unsigned long flags,
1148 unsigned int vm_flags, unsigned long pgoff);
1149
1150static inline unsigned long do_mmap(struct file *file, unsigned long addr,
1151 unsigned long len, unsigned long prot,
1152 unsigned long flag, unsigned long offset)
1153{
1154 unsigned long ret = -EINVAL;
1155 if ((offset + PAGE_ALIGN(len)) < offset)
1156 goto out;
1157 if (!(offset & ~PAGE_MASK))
1158 ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1159out:
1160 return ret;
1161}
1162
1163extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1164
1165extern unsigned long do_brk(unsigned long, unsigned long);
1166
1167
1168extern unsigned long page_unuse(struct page *);
1169extern void truncate_inode_pages(struct address_space *, loff_t);
1170extern void truncate_inode_pages_range(struct address_space *,
1171 loff_t lstart, loff_t lend);
1172
1173
1174extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1175
1176
1177int write_one_page(struct page *page, int wait);
1178void task_dirty_inc(struct task_struct *tsk);
1179
1180
1181#define VM_MAX_READAHEAD 128
1182#define VM_MIN_READAHEAD 16
1183
1184int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
1185 pgoff_t offset, unsigned long nr_to_read);
1186int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1187 pgoff_t offset, unsigned long nr_to_read);
1188
1189void page_cache_sync_readahead(struct address_space *mapping,
1190 struct file_ra_state *ra,
1191 struct file *filp,
1192 pgoff_t offset,
1193 unsigned long size);
1194
1195void page_cache_async_readahead(struct address_space *mapping,
1196 struct file_ra_state *ra,
1197 struct file *filp,
1198 struct page *pg,
1199 pgoff_t offset,
1200 unsigned long size);
1201
1202unsigned long max_sane_readahead(unsigned long nr);
1203
1204
1205extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1206#ifdef CONFIG_IA64
1207extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1208#endif
1209extern int expand_stack_downwards(struct vm_area_struct *vma,
1210 unsigned long address);
1211
1212
1213extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1214extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1215 struct vm_area_struct **pprev);
1216
1217
1218
1219static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1220{
1221 struct vm_area_struct * vma = find_vma(mm,start_addr);
1222
1223 if (vma && end_addr <= vma->vm_start)
1224 vma = NULL;
1225 return vma;
1226}
1227
1228static inline unsigned long vma_pages(struct vm_area_struct *vma)
1229{
1230 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1231}
1232
1233pgprot_t vm_get_page_prot(unsigned long vm_flags);
1234struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
1235int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1236 unsigned long pfn, unsigned long size, pgprot_t);
1237int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
1238int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1239 unsigned long pfn);
1240int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1241 unsigned long pfn);
1242
1243struct page *follow_page(struct vm_area_struct *, unsigned long address,
1244 unsigned int foll_flags);
1245#define FOLL_WRITE 0x01
1246#define FOLL_TOUCH 0x02
1247#define FOLL_GET 0x04
1248#define FOLL_ANON 0x08
1249
1250typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1251 void *data);
1252extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1253 unsigned long size, pte_fn_t fn, void *data);
1254
1255#ifdef CONFIG_PROC_FS
1256void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
1257#else
1258static inline void vm_stat_account(struct mm_struct *mm,
1259 unsigned long flags, struct file *file, long pages)
1260{
1261}
1262#endif
1263
1264#ifdef CONFIG_DEBUG_PAGEALLOC
1265extern int debug_pagealloc_enabled;
1266
1267extern void kernel_map_pages(struct page *page, int numpages, int enable);
1268
1269static inline void enable_debug_pagealloc(void)
1270{
1271 debug_pagealloc_enabled = 1;
1272}
1273#ifdef CONFIG_HIBERNATION
1274extern bool kernel_page_present(struct page *page);
1275#endif
1276#else
1277static inline void
1278kernel_map_pages(struct page *page, int numpages, int enable) {}
1279static inline void enable_debug_pagealloc(void)
1280{
1281}
1282#ifdef CONFIG_HIBERNATION
1283static inline bool kernel_page_present(struct page *page) { return true; }
1284#endif
1285#endif
1286
1287extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk);
1288#ifdef __HAVE_ARCH_GATE_AREA
1289int in_gate_area_no_task(unsigned long addr);
1290int in_gate_area(struct task_struct *task, unsigned long addr);
1291#else
1292int in_gate_area_no_task(unsigned long addr);
1293#define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);})
1294#endif
1295
1296int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *,
1297 void __user *, size_t *, loff_t *);
1298unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
1299 unsigned long lru_pages);
1300
1301#ifndef CONFIG_MMU
1302#define randomize_va_space 0
1303#else
1304extern int randomize_va_space;
1305#endif
1306
1307const char * arch_vma_name(struct vm_area_struct *vma);
1308void print_vma_addr(char *prefix, unsigned long rip);
1309
1310struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
1311pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
1312pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
1313pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
1314pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
1315void *vmemmap_alloc_block(unsigned long size, int node);
1316void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
1317int vmemmap_populate_basepages(struct page *start_page,
1318 unsigned long pages, int node);
1319int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
1320void vmemmap_populate_print_last(void);
1321
1322extern void *alloc_locked_buffer(size_t size);
1323extern void free_locked_buffer(void *buffer, size_t size);
1324extern void release_locked_buffer(void *buffer, size_t size);
1325#endif
1326#endif
1327