1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/gfp.h>
9#include <linux/list.h>
10#include <linux/mmdebug.h>
11#include <linux/mmzone.h>
12#include <linux/rbtree.h>
13#include <linux/prio_tree.h>
14#include <linux/debug_locks.h>
15#include <linux/mm_types.h>
16
17struct mempolicy;
18struct anon_vma;
19struct file_ra_state;
20struct user_struct;
21struct writeback_control;
22
23#ifndef CONFIG_DISCONTIGMEM
24extern unsigned long max_mapnr;
25#endif
26
27extern unsigned long num_physpages;
28extern void * high_memory;
29extern int page_cluster;
30
31#ifdef CONFIG_SYSCTL
32extern int sysctl_legacy_va_layout;
33#else
34#define sysctl_legacy_va_layout 0
35#endif
36
37extern unsigned long mmap_min_addr;
38
39#include <asm/page.h>
40#include <asm/pgtable.h>
41#include <asm/processor.h>
42
43#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
44
45
46#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
47
48
49
50
51
52
53
54
55
56
57extern struct kmem_cache *vm_area_cachep;
58
59#ifndef CONFIG_MMU
60extern struct rb_root nommu_region_tree;
61extern struct rw_semaphore nommu_region_sem;
62
63extern unsigned int kobjsize(const void *objp);
64#endif
65
66
67
68
69#define VM_READ 0x00000001
70#define VM_WRITE 0x00000002
71#define VM_EXEC 0x00000004
72#define VM_SHARED 0x00000008
73
74
75#define VM_MAYREAD 0x00000010
76#define VM_MAYWRITE 0x00000020
77#define VM_MAYEXEC 0x00000040
78#define VM_MAYSHARE 0x00000080
79
80#define VM_GROWSDOWN 0x00000100
81#define VM_GROWSUP 0x00000200
82#define VM_PFNMAP 0x00000400
83#define VM_DENYWRITE 0x00000800
84
85#define VM_EXECUTABLE 0x00001000
86#define VM_LOCKED 0x00002000
87#define VM_IO 0x00004000
88
89
90#define VM_SEQ_READ 0x00008000
91#define VM_RAND_READ 0x00010000
92
93#define VM_DONTCOPY 0x00020000
94#define VM_DONTEXPAND 0x00040000
95#define VM_RESERVED 0x00080000
96#define VM_ACCOUNT 0x00100000
97#define VM_NORESERVE 0x00200000
98#define VM_HUGETLB 0x00400000
99#define VM_NONLINEAR 0x00800000
100#define VM_MAPPED_COPY 0x01000000
101#define VM_INSERTPAGE 0x02000000
102#define VM_ALWAYSDUMP 0x04000000
103
104#define VM_CAN_NONLINEAR 0x08000000
105#define VM_MIXEDMAP 0x10000000
106#define VM_SAO 0x20000000
107
108#ifndef VM_STACK_DEFAULT_FLAGS
109#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
110#endif
111
112#ifdef CONFIG_STACK_GROWSUP
113#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
114#else
115#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
116#endif
117
118#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
119#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
120#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
121#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
122#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
123
124
125
126
127#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_RESERVED | VM_PFNMAP)
128
129
130
131
132
133extern pgprot_t protection_map[16];
134
135#define FAULT_FLAG_WRITE 0x01
136#define FAULT_FLAG_NONLINEAR 0x02
137
138
139
140
141
142
143
144
145
146static inline int is_linear_pfn_mapping(struct vm_area_struct *vma)
147{
148 return ((vma->vm_flags & VM_PFNMAP) && vma->vm_pgoff);
149}
150
151static inline int is_pfn_mapping(struct vm_area_struct *vma)
152{
153 return (vma->vm_flags & VM_PFNMAP);
154}
155
156
157
158
159
160
161
162
163
164
165struct vm_fault {
166 unsigned int flags;
167 pgoff_t pgoff;
168 void __user *virtual_address;
169
170 struct page *page;
171
172
173
174
175};
176
177
178
179
180
181
182struct vm_operations_struct {
183 void (*open)(struct vm_area_struct * area);
184 void (*close)(struct vm_area_struct * area);
185 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
186
187
188
189 int (*page_mkwrite)(struct vm_area_struct *vma, struct page *page);
190
191
192
193
194 int (*access)(struct vm_area_struct *vma, unsigned long addr,
195 void *buf, int len, int write);
196#ifdef CONFIG_NUMA
197
198
199
200
201
202
203
204 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
205
206
207
208
209
210
211
212
213
214
215
216 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
217 unsigned long addr);
218 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
219 const nodemask_t *to, unsigned long flags);
220#endif
221};
222
223struct mmu_gather;
224struct inode;
225
226#define page_private(page) ((page)->private)
227#define set_page_private(page, v) ((page)->private = (v))
228
229
230
231
232
233#include <linux/page-flags.h>
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251static inline int put_page_testzero(struct page *page)
252{
253 VM_BUG_ON(atomic_read(&page->_count) == 0);
254 return atomic_dec_and_test(&page->_count);
255}
256
257
258
259
260
261static inline int get_page_unless_zero(struct page *page)
262{
263 return atomic_inc_not_zero(&page->_count);
264}
265
266
267struct page *vmalloc_to_page(const void *addr);
268unsigned long vmalloc_to_pfn(const void *addr);
269
270
271
272
273
274
275
276static inline int is_vmalloc_addr(const void *x)
277{
278#ifdef CONFIG_MMU
279 unsigned long addr = (unsigned long)x;
280
281 return addr >= VMALLOC_START && addr < VMALLOC_END;
282#else
283 return 0;
284#endif
285}
286
287static inline struct page *compound_head(struct page *page)
288{
289 if (unlikely(PageTail(page)))
290 return page->first_page;
291 return page;
292}
293
294static inline int page_count(struct page *page)
295{
296 return atomic_read(&compound_head(page)->_count);
297}
298
299static inline void get_page(struct page *page)
300{
301 page = compound_head(page);
302 VM_BUG_ON(atomic_read(&page->_count) == 0);
303 atomic_inc(&page->_count);
304}
305
306static inline struct page *virt_to_head_page(const void *x)
307{
308 struct page *page = virt_to_page(x);
309 return compound_head(page);
310}
311
312
313
314
315
316static inline void init_page_count(struct page *page)
317{
318 atomic_set(&page->_count, 1);
319}
320
321void put_page(struct page *page);
322void put_pages_list(struct list_head *pages);
323
324void split_page(struct page *page, unsigned int order);
325
326
327
328
329
330
331typedef void compound_page_dtor(struct page *);
332
333static inline void set_compound_page_dtor(struct page *page,
334 compound_page_dtor *dtor)
335{
336 page[1].lru.next = (void *)dtor;
337}
338
339static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
340{
341 return (compound_page_dtor *)page[1].lru.next;
342}
343
344static inline int compound_order(struct page *page)
345{
346 if (!PageHead(page))
347 return 0;
348 return (unsigned long)page[1].lru.prev;
349}
350
351static inline void set_compound_order(struct page *page, unsigned long order)
352{
353 page[1].lru.prev = (void *)order;
354}
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
437#define SECTIONS_WIDTH SECTIONS_SHIFT
438#else
439#define SECTIONS_WIDTH 0
440#endif
441
442#define ZONES_WIDTH ZONES_SHIFT
443
444#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
445#define NODES_WIDTH NODES_SHIFT
446#else
447#ifdef CONFIG_SPARSEMEM_VMEMMAP
448#error "Vmemmap: No space for nodes field in page flags"
449#endif
450#define NODES_WIDTH 0
451#endif
452
453
454#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
455#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
456#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
457
458
459
460
461
462#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
463#define NODE_NOT_IN_PAGE_FLAGS
464#endif
465
466#ifndef PFN_SECTION_SHIFT
467#define PFN_SECTION_SHIFT 0
468#endif
469
470
471
472
473
474
475#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
476#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
477#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
478
479
480#ifdef NODE_NOT_IN_PAGEFLAGS
481#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
482#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
483 SECTIONS_PGOFF : ZONES_PGOFF)
484#else
485#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
486#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
487 NODES_PGOFF : ZONES_PGOFF)
488#endif
489
490#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
491
492#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
493#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
494#endif
495
496#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
497#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
498#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
499#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
500
501static inline enum zone_type page_zonenum(struct page *page)
502{
503 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
504}
505
506
507
508
509
510
511
512
513
514static inline int page_zone_id(struct page *page)
515{
516 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
517}
518
519static inline int zone_to_nid(struct zone *zone)
520{
521#ifdef CONFIG_NUMA
522 return zone->node;
523#else
524 return 0;
525#endif
526}
527
528#ifdef NODE_NOT_IN_PAGE_FLAGS
529extern int page_to_nid(struct page *page);
530#else
531static inline int page_to_nid(struct page *page)
532{
533 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
534}
535#endif
536
537static inline struct zone *page_zone(struct page *page)
538{
539 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
540}
541
542#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
543static inline unsigned long page_to_section(struct page *page)
544{
545 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
546}
547#endif
548
549static inline void set_page_zone(struct page *page, enum zone_type zone)
550{
551 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
552 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
553}
554
555static inline void set_page_node(struct page *page, unsigned long node)
556{
557 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
558 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
559}
560
561static inline void set_page_section(struct page *page, unsigned long section)
562{
563 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
564 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
565}
566
567static inline void set_page_links(struct page *page, enum zone_type zone,
568 unsigned long node, unsigned long pfn)
569{
570 set_page_zone(page, zone);
571 set_page_node(page, node);
572 set_page_section(page, pfn_to_section_nr(pfn));
573}
574
575
576
577
578
579static inline unsigned long round_hint_to_min(unsigned long hint)
580{
581#ifdef CONFIG_SECURITY
582 hint &= PAGE_MASK;
583 if (((void *)hint != NULL) &&
584 (hint < mmap_min_addr))
585 return PAGE_ALIGN(mmap_min_addr);
586#endif
587 return hint;
588}
589
590
591
592
593#include <linux/vmstat.h>
594
595static __always_inline void *lowmem_page_address(struct page *page)
596{
597 return __va(page_to_pfn(page) << PAGE_SHIFT);
598}
599
600#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
601#define HASHED_PAGE_VIRTUAL
602#endif
603
604#if defined(WANT_PAGE_VIRTUAL)
605#define page_address(page) ((page)->virtual)
606#define set_page_address(page, address) \
607 do { \
608 (page)->virtual = (address); \
609 } while(0)
610#define page_address_init() do { } while(0)
611#endif
612
613#if defined(HASHED_PAGE_VIRTUAL)
614void *page_address(struct page *page);
615void set_page_address(struct page *page, void *virtual);
616void page_address_init(void);
617#endif
618
619#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
620#define page_address(page) lowmem_page_address(page)
621#define set_page_address(page, address) do { } while(0)
622#define page_address_init() do { } while(0)
623#endif
624
625
626
627
628
629
630
631
632
633
634#define PAGE_MAPPING_ANON 1
635
636extern struct address_space swapper_space;
637static inline struct address_space *page_mapping(struct page *page)
638{
639 struct address_space *mapping = page->mapping;
640
641 VM_BUG_ON(PageSlab(page));
642#ifdef CONFIG_SWAP
643 if (unlikely(PageSwapCache(page)))
644 mapping = &swapper_space;
645 else
646#endif
647 if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
648 mapping = NULL;
649 return mapping;
650}
651
652static inline int PageAnon(struct page *page)
653{
654 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
655}
656
657
658
659
660
661static inline pgoff_t page_index(struct page *page)
662{
663 if (unlikely(PageSwapCache(page)))
664 return page_private(page);
665 return page->index;
666}
667
668
669
670
671
672
673static inline void reset_page_mapcount(struct page *page)
674{
675 atomic_set(&(page)->_mapcount, -1);
676}
677
678static inline int page_mapcount(struct page *page)
679{
680 return atomic_read(&(page)->_mapcount) + 1;
681}
682
683
684
685
686static inline int page_mapped(struct page *page)
687{
688 return atomic_read(&(page)->_mapcount) >= 0;
689}
690
691
692
693
694
695
696
697#define VM_FAULT_MINOR 0
698
699#define VM_FAULT_OOM 0x0001
700#define VM_FAULT_SIGBUS 0x0002
701#define VM_FAULT_MAJOR 0x0004
702#define VM_FAULT_WRITE 0x0008
703
704#define VM_FAULT_NOPAGE 0x0100
705#define VM_FAULT_LOCKED 0x0200
706
707#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS)
708
709
710
711
712extern void pagefault_out_of_memory(void);
713
714#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
715
716extern void show_free_areas(void);
717
718#ifdef CONFIG_SHMEM
719extern int shmem_lock(struct file *file, int lock, struct user_struct *user);
720#else
721static inline int shmem_lock(struct file *file, int lock,
722 struct user_struct *user)
723{
724 return 0;
725}
726#endif
727struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
728
729int shmem_zero_setup(struct vm_area_struct *);
730
731#ifndef CONFIG_MMU
732extern unsigned long shmem_get_unmapped_area(struct file *file,
733 unsigned long addr,
734 unsigned long len,
735 unsigned long pgoff,
736 unsigned long flags);
737#endif
738
739extern int can_do_mlock(void);
740extern int user_shm_lock(size_t, struct user_struct *);
741extern void user_shm_unlock(size_t, struct user_struct *);
742
743
744
745
746struct zap_details {
747 struct vm_area_struct *nonlinear_vma;
748 struct address_space *check_mapping;
749 pgoff_t first_index;
750 pgoff_t last_index;
751 spinlock_t *i_mmap_lock;
752 unsigned long truncate_count;
753};
754
755struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
756 pte_t pte);
757
758int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
759 unsigned long size);
760unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
761 unsigned long size, struct zap_details *);
762unsigned long unmap_vmas(struct mmu_gather **tlb,
763 struct vm_area_struct *start_vma, unsigned long start_addr,
764 unsigned long end_addr, unsigned long *nr_accounted,
765 struct zap_details *);
766
767
768
769
770
771
772
773
774
775
776
777struct mm_walk {
778 int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
779 int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
780 int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
781 int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
782 int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
783 struct mm_struct *mm;
784 void *private;
785};
786
787int walk_page_range(unsigned long addr, unsigned long end,
788 struct mm_walk *walk);
789void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
790 unsigned long end, unsigned long floor, unsigned long ceiling);
791int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
792 struct vm_area_struct *vma);
793void unmap_mapping_range(struct address_space *mapping,
794 loff_t const holebegin, loff_t const holelen, int even_cows);
795int follow_phys(struct vm_area_struct *vma, unsigned long address,
796 unsigned int flags, unsigned long *prot, resource_size_t *phys);
797int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
798 void *buf, int len, int write);
799
800static inline void unmap_shared_mapping_range(struct address_space *mapping,
801 loff_t const holebegin, loff_t const holelen)
802{
803 unmap_mapping_range(mapping, holebegin, holelen, 0);
804}
805
806extern int vmtruncate(struct inode * inode, loff_t offset);
807extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end);
808
809#ifdef CONFIG_MMU
810extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
811 unsigned long address, int write_access);
812#else
813static inline int handle_mm_fault(struct mm_struct *mm,
814 struct vm_area_struct *vma, unsigned long address,
815 int write_access)
816{
817
818 BUG();
819 return VM_FAULT_SIGBUS;
820}
821#endif
822
823extern int make_pages_present(unsigned long addr, unsigned long end);
824extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
825
826int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start,
827 int len, int write, int force, struct page **pages, struct vm_area_struct **vmas);
828
829extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
830extern void do_invalidatepage(struct page *page, unsigned long offset);
831
832int __set_page_dirty_nobuffers(struct page *page);
833int __set_page_dirty_no_writeback(struct page *page);
834int redirty_page_for_writepage(struct writeback_control *wbc,
835 struct page *page);
836int set_page_dirty(struct page *page);
837int set_page_dirty_lock(struct page *page);
838int clear_page_dirty_for_io(struct page *page);
839
840extern unsigned long move_page_tables(struct vm_area_struct *vma,
841 unsigned long old_addr, struct vm_area_struct *new_vma,
842 unsigned long new_addr, unsigned long len);
843extern unsigned long do_mremap(unsigned long addr,
844 unsigned long old_len, unsigned long new_len,
845 unsigned long flags, unsigned long new_addr);
846extern int mprotect_fixup(struct vm_area_struct *vma,
847 struct vm_area_struct **pprev, unsigned long start,
848 unsigned long end, unsigned long newflags);
849
850
851
852
853
854
855
856
857
858
859
860int get_user_pages_fast(unsigned long start, int nr_pages, int write,
861 struct page **pages);
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878struct shrinker {
879 int (*shrink)(int nr_to_scan, gfp_t gfp_mask);
880 int seeks;
881
882
883 struct list_head list;
884 long nr;
885};
886#define DEFAULT_SEEKS 2
887extern void register_shrinker(struct shrinker *);
888extern void unregister_shrinker(struct shrinker *);
889
890int vma_wants_writenotify(struct vm_area_struct *vma);
891
892extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
893
894#ifdef __PAGETABLE_PUD_FOLDED
895static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
896 unsigned long address)
897{
898 return 0;
899}
900#else
901int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
902#endif
903
904#ifdef __PAGETABLE_PMD_FOLDED
905static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
906 unsigned long address)
907{
908 return 0;
909}
910#else
911int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
912#endif
913
914int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
915int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
916
917
918
919
920
921#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
922static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
923{
924 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
925 NULL: pud_offset(pgd, address);
926}
927
928static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
929{
930 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
931 NULL: pmd_offset(pud, address);
932}
933#endif
934
935#if USE_SPLIT_PTLOCKS
936
937
938
939
940
941
942#define __pte_lockptr(page) &((page)->ptl)
943#define pte_lock_init(_page) do { \
944 spin_lock_init(__pte_lockptr(_page)); \
945} while (0)
946#define pte_lock_deinit(page) ((page)->mapping = NULL)
947#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
948#else
949
950
951
952#define pte_lock_init(page) do {} while (0)
953#define pte_lock_deinit(page) do {} while (0)
954#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
955#endif
956
957static inline void pgtable_page_ctor(struct page *page)
958{
959 pte_lock_init(page);
960 inc_zone_page_state(page, NR_PAGETABLE);
961}
962
963static inline void pgtable_page_dtor(struct page *page)
964{
965 pte_lock_deinit(page);
966 dec_zone_page_state(page, NR_PAGETABLE);
967}
968
969#define pte_offset_map_lock(mm, pmd, address, ptlp) \
970({ \
971 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
972 pte_t *__pte = pte_offset_map(pmd, address); \
973 *(ptlp) = __ptl; \
974 spin_lock(__ptl); \
975 __pte; \
976})
977
978#define pte_unmap_unlock(pte, ptl) do { \
979 spin_unlock(ptl); \
980 pte_unmap(pte); \
981} while (0)
982
983#define pte_alloc_map(mm, pmd, address) \
984 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
985 NULL: pte_offset_map(pmd, address))
986
987#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
988 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
989 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
990
991#define pte_alloc_kernel(pmd, address) \
992 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
993 NULL: pte_offset_kernel(pmd, address))
994
995extern void free_area_init(unsigned long * zones_size);
996extern void free_area_init_node(int nid, unsigned long * zones_size,
997 unsigned long zone_start_pfn, unsigned long *zholes_size);
998#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1027extern void add_active_range(unsigned int nid, unsigned long start_pfn,
1028 unsigned long end_pfn);
1029extern void remove_active_range(unsigned int nid, unsigned long start_pfn,
1030 unsigned long end_pfn);
1031extern void push_node_boundaries(unsigned int nid, unsigned long start_pfn,
1032 unsigned long end_pfn);
1033extern void remove_all_active_ranges(void);
1034extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1035 unsigned long end_pfn);
1036extern void get_pfn_range_for_nid(unsigned int nid,
1037 unsigned long *start_pfn, unsigned long *end_pfn);
1038extern unsigned long find_min_pfn_with_active_regions(void);
1039extern void free_bootmem_with_active_regions(int nid,
1040 unsigned long max_low_pfn);
1041typedef int (*work_fn_t)(unsigned long, unsigned long, void *);
1042extern void work_with_active_regions(int nid, work_fn_t work_fn, void *data);
1043extern void sparse_memory_present_with_active_regions(int nid);
1044#endif
1045
1046#if !defined(CONFIG_ARCH_POPULATES_NODE_MAP) && \
1047 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1048static inline int __early_pfn_to_nid(unsigned long pfn)
1049{
1050 return 0;
1051}
1052#else
1053
1054extern int __meminit early_pfn_to_nid(unsigned long pfn);
1055#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1056
1057extern int __meminit __early_pfn_to_nid(unsigned long pfn);
1058#endif
1059#endif
1060
1061extern void set_dma_reserve(unsigned long new_dma_reserve);
1062extern void memmap_init_zone(unsigned long, int, unsigned long,
1063 unsigned long, enum memmap_context);
1064extern void setup_per_zone_pages_min(void);
1065extern void mem_init(void);
1066extern void __init mmap_init(void);
1067extern void show_mem(void);
1068extern void si_meminfo(struct sysinfo * val);
1069extern void si_meminfo_node(struct sysinfo *val, int nid);
1070extern int after_bootmem;
1071
1072#ifdef CONFIG_NUMA
1073extern void setup_per_cpu_pageset(void);
1074#else
1075static inline void setup_per_cpu_pageset(void) {}
1076#endif
1077
1078
1079extern atomic_t mmap_pages_allocated;
1080
1081
1082void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
1083void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
1084void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *);
1085struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma,
1086 struct prio_tree_iter *iter);
1087
1088#define vma_prio_tree_foreach(vma, iter, root, begin, end) \
1089 for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \
1090 (vma = vma_prio_tree_next(vma, iter)); )
1091
1092static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1093 struct list_head *list)
1094{
1095 vma->shared.vm_set.parent = NULL;
1096 list_add_tail(&vma->shared.vm_set.list, list);
1097}
1098
1099
1100extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1101extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
1102 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1103extern struct vm_area_struct *vma_merge(struct mm_struct *,
1104 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1105 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1106 struct mempolicy *);
1107extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1108extern int split_vma(struct mm_struct *,
1109 struct vm_area_struct *, unsigned long addr, int new_below);
1110extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1111extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1112 struct rb_node **, struct rb_node *);
1113extern void unlink_file_vma(struct vm_area_struct *);
1114extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1115 unsigned long addr, unsigned long len, pgoff_t pgoff);
1116extern void exit_mmap(struct mm_struct *);
1117
1118extern int mm_take_all_locks(struct mm_struct *mm);
1119extern void mm_drop_all_locks(struct mm_struct *mm);
1120
1121#ifdef CONFIG_PROC_FS
1122
1123extern void added_exe_file_vma(struct mm_struct *mm);
1124extern void removed_exe_file_vma(struct mm_struct *mm);
1125#else
1126static inline void added_exe_file_vma(struct mm_struct *mm)
1127{}
1128
1129static inline void removed_exe_file_vma(struct mm_struct *mm)
1130{}
1131#endif
1132
1133extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1134extern int install_special_mapping(struct mm_struct *mm,
1135 unsigned long addr, unsigned long len,
1136 unsigned long flags, struct page **pages);
1137
1138extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1139
1140extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1141 unsigned long len, unsigned long prot,
1142 unsigned long flag, unsigned long pgoff);
1143extern unsigned long mmap_region(struct file *file, unsigned long addr,
1144 unsigned long len, unsigned long flags,
1145 unsigned int vm_flags, unsigned long pgoff);
1146
1147static inline unsigned long do_mmap(struct file *file, unsigned long addr,
1148 unsigned long len, unsigned long prot,
1149 unsigned long flag, unsigned long offset)
1150{
1151 unsigned long ret = -EINVAL;
1152 if ((offset + PAGE_ALIGN(len)) < offset)
1153 goto out;
1154 if (!(offset & ~PAGE_MASK))
1155 ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1156out:
1157 return ret;
1158}
1159
1160extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1161
1162extern unsigned long do_brk(unsigned long, unsigned long);
1163
1164
1165extern unsigned long page_unuse(struct page *);
1166extern void truncate_inode_pages(struct address_space *, loff_t);
1167extern void truncate_inode_pages_range(struct address_space *,
1168 loff_t lstart, loff_t lend);
1169
1170
1171extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1172
1173
1174int write_one_page(struct page *page, int wait);
1175void task_dirty_inc(struct task_struct *tsk);
1176
1177
1178#define VM_MAX_READAHEAD 128
1179#define VM_MIN_READAHEAD 16
1180
1181int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
1182 pgoff_t offset, unsigned long nr_to_read);
1183int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1184 pgoff_t offset, unsigned long nr_to_read);
1185
1186void page_cache_sync_readahead(struct address_space *mapping,
1187 struct file_ra_state *ra,
1188 struct file *filp,
1189 pgoff_t offset,
1190 unsigned long size);
1191
1192void page_cache_async_readahead(struct address_space *mapping,
1193 struct file_ra_state *ra,
1194 struct file *filp,
1195 struct page *pg,
1196 pgoff_t offset,
1197 unsigned long size);
1198
1199unsigned long max_sane_readahead(unsigned long nr);
1200
1201
1202extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1203#ifdef CONFIG_IA64
1204extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1205#endif
1206extern int expand_stack_downwards(struct vm_area_struct *vma,
1207 unsigned long address);
1208
1209
1210extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1211extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1212 struct vm_area_struct **pprev);
1213
1214
1215
1216static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1217{
1218 struct vm_area_struct * vma = find_vma(mm,start_addr);
1219
1220 if (vma && end_addr <= vma->vm_start)
1221 vma = NULL;
1222 return vma;
1223}
1224
1225static inline unsigned long vma_pages(struct vm_area_struct *vma)
1226{
1227 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1228}
1229
1230pgprot_t vm_get_page_prot(unsigned long vm_flags);
1231struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
1232int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1233 unsigned long pfn, unsigned long size, pgprot_t);
1234int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
1235int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1236 unsigned long pfn);
1237int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1238 unsigned long pfn);
1239
1240struct page *follow_page(struct vm_area_struct *, unsigned long address,
1241 unsigned int foll_flags);
1242#define FOLL_WRITE 0x01
1243#define FOLL_TOUCH 0x02
1244#define FOLL_GET 0x04
1245#define FOLL_ANON 0x08
1246
1247typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1248 void *data);
1249extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1250 unsigned long size, pte_fn_t fn, void *data);
1251
1252#ifdef CONFIG_PROC_FS
1253void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
1254#else
1255static inline void vm_stat_account(struct mm_struct *mm,
1256 unsigned long flags, struct file *file, long pages)
1257{
1258}
1259#endif
1260
1261#ifdef CONFIG_DEBUG_PAGEALLOC
1262extern int debug_pagealloc_enabled;
1263
1264extern void kernel_map_pages(struct page *page, int numpages, int enable);
1265
1266static inline void enable_debug_pagealloc(void)
1267{
1268 debug_pagealloc_enabled = 1;
1269}
1270#ifdef CONFIG_HIBERNATION
1271extern bool kernel_page_present(struct page *page);
1272#endif
1273#else
1274static inline void
1275kernel_map_pages(struct page *page, int numpages, int enable) {}
1276static inline void enable_debug_pagealloc(void)
1277{
1278}
1279#ifdef CONFIG_HIBERNATION
1280static inline bool kernel_page_present(struct page *page) { return true; }
1281#endif
1282#endif
1283
1284extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk);
1285#ifdef __HAVE_ARCH_GATE_AREA
1286int in_gate_area_no_task(unsigned long addr);
1287int in_gate_area(struct task_struct *task, unsigned long addr);
1288#else
1289int in_gate_area_no_task(unsigned long addr);
1290#define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);})
1291#endif
1292
1293int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *,
1294 void __user *, size_t *, loff_t *);
1295unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
1296 unsigned long lru_pages);
1297
1298#ifndef CONFIG_MMU
1299#define randomize_va_space 0
1300#else
1301extern int randomize_va_space;
1302#endif
1303
1304const char * arch_vma_name(struct vm_area_struct *vma);
1305void print_vma_addr(char *prefix, unsigned long rip);
1306
1307struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
1308pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
1309pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
1310pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
1311pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
1312void *vmemmap_alloc_block(unsigned long size, int node);
1313void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
1314int vmemmap_populate_basepages(struct page *start_page,
1315 unsigned long pages, int node);
1316int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
1317void vmemmap_populate_print_last(void);
1318
1319extern void *alloc_locked_buffer(size_t size);
1320extern void free_locked_buffer(void *buffer, size_t size);
1321extern void release_locked_buffer(void *buffer, size_t size);
1322#endif
1323#endif
1324