1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/gfp.h>
9#include <linux/list.h>
10#include <linux/mmzone.h>
11#include <linux/rbtree.h>
12#include <linux/prio_tree.h>
13#include <linux/debug_locks.h>
14#include <linux/mm_types.h>
15
16struct mempolicy;
17struct anon_vma;
18struct file_ra_state;
19struct user_struct;
20struct writeback_control;
21
22#ifndef CONFIG_DISCONTIGMEM
23extern unsigned long max_mapnr;
24#endif
25
26extern unsigned long num_physpages;
27extern void * high_memory;
28extern int page_cluster;
29
30#ifdef CONFIG_SYSCTL
31extern int sysctl_legacy_va_layout;
32#else
33#define sysctl_legacy_va_layout 0
34#endif
35
36extern unsigned long mmap_min_addr;
37
38#include <asm/page.h>
39#include <asm/pgtable.h>
40#include <asm/processor.h>
41
42#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
43
44
45
46
47
48
49
50
51
52
53extern struct kmem_cache *vm_area_cachep;
54
55
56
57
58
59
60struct vm_list_struct {
61 struct vm_list_struct *next;
62 struct vm_area_struct *vma;
63};
64
65#ifndef CONFIG_MMU
66extern struct rb_root nommu_vma_tree;
67extern struct rw_semaphore nommu_vma_sem;
68
69extern unsigned int kobjsize(const void *objp);
70#endif
71
72
73
74
75#define VM_READ 0x00000001
76#define VM_WRITE 0x00000002
77#define VM_EXEC 0x00000004
78#define VM_SHARED 0x00000008
79
80
81#define VM_MAYREAD 0x00000010
82#define VM_MAYWRITE 0x00000020
83#define VM_MAYEXEC 0x00000040
84#define VM_MAYSHARE 0x00000080
85
86#define VM_GROWSDOWN 0x00000100
87#define VM_GROWSUP 0x00000200
88#define VM_PFNMAP 0x00000400
89#define VM_DENYWRITE 0x00000800
90
91#define VM_EXECUTABLE 0x00001000
92#define VM_LOCKED 0x00002000
93#define VM_IO 0x00004000
94
95
96#define VM_SEQ_READ 0x00008000
97#define VM_RAND_READ 0x00010000
98
99#define VM_DONTCOPY 0x00020000
100#define VM_DONTEXPAND 0x00040000
101#define VM_RESERVED 0x00080000
102#define VM_ACCOUNT 0x00100000
103#define VM_HUGETLB 0x00400000
104#define VM_NONLINEAR 0x00800000
105#define VM_MAPPED_COPY 0x01000000
106#define VM_INSERTPAGE 0x02000000
107#define VM_ALWAYSDUMP 0x04000000
108
109#define VM_CAN_NONLINEAR 0x08000000
110#define VM_MIXEDMAP 0x10000000
111
112#ifndef VM_STACK_DEFAULT_FLAGS
113#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
114#endif
115
116#ifdef CONFIG_STACK_GROWSUP
117#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
118#else
119#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
120#endif
121
122#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
123#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
124#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
125#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
126#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
127
128
129
130
131
132extern pgprot_t protection_map[16];
133
134#define FAULT_FLAG_WRITE 0x01
135#define FAULT_FLAG_NONLINEAR 0x02
136
137
138
139
140
141
142
143
144
145
146
147struct vm_fault {
148 unsigned int flags;
149 pgoff_t pgoff;
150 void __user *virtual_address;
151
152 struct page *page;
153
154
155
156
157};
158
159
160
161
162
163
164struct vm_operations_struct {
165 void (*open)(struct vm_area_struct * area);
166 void (*close)(struct vm_area_struct * area);
167 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
168 unsigned long (*nopfn)(struct vm_area_struct *area,
169 unsigned long address);
170
171
172
173 int (*page_mkwrite)(struct vm_area_struct *vma, struct page *page);
174#ifdef CONFIG_NUMA
175
176
177
178
179
180
181
182 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
183
184
185
186
187
188
189
190
191
192
193
194 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
195 unsigned long addr);
196 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
197 const nodemask_t *to, unsigned long flags);
198#endif
199};
200
201struct mmu_gather;
202struct inode;
203
204#define page_private(page) ((page)->private)
205#define set_page_private(page, v) ((page)->private = (v))
206
207
208
209
210
211#include <linux/page-flags.h>
212
213#ifdef CONFIG_DEBUG_VM
214#define VM_BUG_ON(cond) BUG_ON(cond)
215#else
216#define VM_BUG_ON(condition) do { } while(0)
217#endif
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235static inline int put_page_testzero(struct page *page)
236{
237 VM_BUG_ON(atomic_read(&page->_count) == 0);
238 return atomic_dec_and_test(&page->_count);
239}
240
241
242
243
244
245static inline int get_page_unless_zero(struct page *page)
246{
247 VM_BUG_ON(PageTail(page));
248 return atomic_inc_not_zero(&page->_count);
249}
250
251
252struct page *vmalloc_to_page(const void *addr);
253unsigned long vmalloc_to_pfn(const void *addr);
254
255
256
257
258
259
260
261static inline int is_vmalloc_addr(const void *x)
262{
263#ifdef CONFIG_MMU
264 unsigned long addr = (unsigned long)x;
265
266 return addr >= VMALLOC_START && addr < VMALLOC_END;
267#else
268 return 0;
269#endif
270}
271
272static inline struct page *compound_head(struct page *page)
273{
274 if (unlikely(PageTail(page)))
275 return page->first_page;
276 return page;
277}
278
279static inline int page_count(struct page *page)
280{
281 return atomic_read(&compound_head(page)->_count);
282}
283
284static inline void get_page(struct page *page)
285{
286 page = compound_head(page);
287 VM_BUG_ON(atomic_read(&page->_count) == 0);
288 atomic_inc(&page->_count);
289}
290
291static inline struct page *virt_to_head_page(const void *x)
292{
293 struct page *page = virt_to_page(x);
294 return compound_head(page);
295}
296
297
298
299
300
301static inline void init_page_count(struct page *page)
302{
303 atomic_set(&page->_count, 1);
304}
305
306void put_page(struct page *page);
307void put_pages_list(struct list_head *pages);
308
309void split_page(struct page *page, unsigned int order);
310
311
312
313
314
315
316typedef void compound_page_dtor(struct page *);
317
318static inline void set_compound_page_dtor(struct page *page,
319 compound_page_dtor *dtor)
320{
321 page[1].lru.next = (void *)dtor;
322}
323
324static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
325{
326 return (compound_page_dtor *)page[1].lru.next;
327}
328
329static inline int compound_order(struct page *page)
330{
331 if (!PageHead(page))
332 return 0;
333 return (unsigned long)page[1].lru.prev;
334}
335
336static inline void set_compound_order(struct page *page, unsigned long order)
337{
338 page[1].lru.prev = (void *)order;
339}
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
422#define SECTIONS_WIDTH SECTIONS_SHIFT
423#else
424#define SECTIONS_WIDTH 0
425#endif
426
427#define ZONES_WIDTH ZONES_SHIFT
428
429#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
430#define NODES_WIDTH NODES_SHIFT
431#else
432#ifdef CONFIG_SPARSEMEM_VMEMMAP
433#error "Vmemmap: No space for nodes field in page flags"
434#endif
435#define NODES_WIDTH 0
436#endif
437
438
439#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
440#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
441#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
442
443
444
445
446
447#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
448#define NODE_NOT_IN_PAGE_FLAGS
449#endif
450
451#ifndef PFN_SECTION_SHIFT
452#define PFN_SECTION_SHIFT 0
453#endif
454
455
456
457
458
459
460#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
461#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
462#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
463
464
465#ifdef NODE_NOT_IN_PAGEFLAGS
466#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
467#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
468 SECTIONS_PGOFF : ZONES_PGOFF)
469#else
470#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
471#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
472 NODES_PGOFF : ZONES_PGOFF)
473#endif
474
475#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
476
477#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
478#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
479#endif
480
481#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
482#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
483#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
484#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
485
486static inline enum zone_type page_zonenum(struct page *page)
487{
488 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
489}
490
491
492
493
494
495
496
497
498
499static inline int page_zone_id(struct page *page)
500{
501 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
502}
503
504static inline int zone_to_nid(struct zone *zone)
505{
506#ifdef CONFIG_NUMA
507 return zone->node;
508#else
509 return 0;
510#endif
511}
512
513#ifdef NODE_NOT_IN_PAGE_FLAGS
514extern int page_to_nid(struct page *page);
515#else
516static inline int page_to_nid(struct page *page)
517{
518 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
519}
520#endif
521
522static inline struct zone *page_zone(struct page *page)
523{
524 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
525}
526
527#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
528static inline unsigned long page_to_section(struct page *page)
529{
530 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
531}
532#endif
533
534static inline void set_page_zone(struct page *page, enum zone_type zone)
535{
536 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
537 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
538}
539
540static inline void set_page_node(struct page *page, unsigned long node)
541{
542 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
543 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
544}
545
546static inline void set_page_section(struct page *page, unsigned long section)
547{
548 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
549 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
550}
551
552static inline void set_page_links(struct page *page, enum zone_type zone,
553 unsigned long node, unsigned long pfn)
554{
555 set_page_zone(page, zone);
556 set_page_node(page, node);
557 set_page_section(page, pfn_to_section_nr(pfn));
558}
559
560
561
562
563
564static inline unsigned long round_hint_to_min(unsigned long hint)
565{
566#ifdef CONFIG_SECURITY
567 hint &= PAGE_MASK;
568 if (((void *)hint != NULL) &&
569 (hint < mmap_min_addr))
570 return PAGE_ALIGN(mmap_min_addr);
571#endif
572 return hint;
573}
574
575
576
577
578#include <linux/vmstat.h>
579
580static __always_inline void *lowmem_page_address(struct page *page)
581{
582 return __va(page_to_pfn(page) << PAGE_SHIFT);
583}
584
585#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
586#define HASHED_PAGE_VIRTUAL
587#endif
588
589#if defined(WANT_PAGE_VIRTUAL)
590#define page_address(page) ((page)->virtual)
591#define set_page_address(page, address) \
592 do { \
593 (page)->virtual = (address); \
594 } while(0)
595#define page_address_init() do { } while(0)
596#endif
597
598#if defined(HASHED_PAGE_VIRTUAL)
599void *page_address(struct page *page);
600void set_page_address(struct page *page, void *virtual);
601void page_address_init(void);
602#endif
603
604#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
605#define page_address(page) lowmem_page_address(page)
606#define set_page_address(page, address) do { } while(0)
607#define page_address_init() do { } while(0)
608#endif
609
610
611
612
613
614
615
616
617
618
619#define PAGE_MAPPING_ANON 1
620
621extern struct address_space swapper_space;
622static inline struct address_space *page_mapping(struct page *page)
623{
624 struct address_space *mapping = page->mapping;
625
626 VM_BUG_ON(PageSlab(page));
627#ifdef CONFIG_SWAP
628 if (unlikely(PageSwapCache(page)))
629 mapping = &swapper_space;
630 else
631#endif
632 if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
633 mapping = NULL;
634 return mapping;
635}
636
637static inline int PageAnon(struct page *page)
638{
639 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
640}
641
642
643
644
645
646static inline pgoff_t page_index(struct page *page)
647{
648 if (unlikely(PageSwapCache(page)))
649 return page_private(page);
650 return page->index;
651}
652
653
654
655
656
657
658static inline void reset_page_mapcount(struct page *page)
659{
660 atomic_set(&(page)->_mapcount, -1);
661}
662
663static inline int page_mapcount(struct page *page)
664{
665 return atomic_read(&(page)->_mapcount) + 1;
666}
667
668
669
670
671static inline int page_mapped(struct page *page)
672{
673 return atomic_read(&(page)->_mapcount) >= 0;
674}
675
676
677
678
679#define NOPFN_SIGBUS ((unsigned long) -1)
680#define NOPFN_OOM ((unsigned long) -2)
681#define NOPFN_REFAULT ((unsigned long) -3)
682
683
684
685
686
687
688
689#define VM_FAULT_MINOR 0
690
691#define VM_FAULT_OOM 0x0001
692#define VM_FAULT_SIGBUS 0x0002
693#define VM_FAULT_MAJOR 0x0004
694#define VM_FAULT_WRITE 0x0008
695
696#define VM_FAULT_NOPAGE 0x0100
697#define VM_FAULT_LOCKED 0x0200
698
699#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS)
700
701#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
702
703extern void show_free_areas(void);
704
705#ifdef CONFIG_SHMEM
706int shmem_lock(struct file *file, int lock, struct user_struct *user);
707#else
708static inline int shmem_lock(struct file *file, int lock,
709 struct user_struct *user)
710{
711 return 0;
712}
713#endif
714struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
715
716int shmem_zero_setup(struct vm_area_struct *);
717
718#ifndef CONFIG_MMU
719extern unsigned long shmem_get_unmapped_area(struct file *file,
720 unsigned long addr,
721 unsigned long len,
722 unsigned long pgoff,
723 unsigned long flags);
724#endif
725
726extern int can_do_mlock(void);
727extern int user_shm_lock(size_t, struct user_struct *);
728extern void user_shm_unlock(size_t, struct user_struct *);
729
730
731
732
733struct zap_details {
734 struct vm_area_struct *nonlinear_vma;
735 struct address_space *check_mapping;
736 pgoff_t first_index;
737 pgoff_t last_index;
738 spinlock_t *i_mmap_lock;
739 unsigned long truncate_count;
740};
741
742struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
743 pte_t pte);
744
745unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
746 unsigned long size, struct zap_details *);
747unsigned long unmap_vmas(struct mmu_gather **tlb,
748 struct vm_area_struct *start_vma, unsigned long start_addr,
749 unsigned long end_addr, unsigned long *nr_accounted,
750 struct zap_details *);
751
752
753
754
755
756
757
758
759
760
761
762struct mm_walk {
763 int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
764 int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
765 int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
766 int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
767 int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
768 struct mm_struct *mm;
769 void *private;
770};
771
772int walk_page_range(unsigned long addr, unsigned long end,
773 struct mm_walk *walk);
774void free_pgd_range(struct mmu_gather **tlb, unsigned long addr,
775 unsigned long end, unsigned long floor, unsigned long ceiling);
776void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *start_vma,
777 unsigned long floor, unsigned long ceiling);
778int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
779 struct vm_area_struct *vma);
780void unmap_mapping_range(struct address_space *mapping,
781 loff_t const holebegin, loff_t const holelen, int even_cows);
782
783static inline void unmap_shared_mapping_range(struct address_space *mapping,
784 loff_t const holebegin, loff_t const holelen)
785{
786 unmap_mapping_range(mapping, holebegin, holelen, 0);
787}
788
789extern int vmtruncate(struct inode * inode, loff_t offset);
790extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end);
791
792#ifdef CONFIG_MMU
793extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
794 unsigned long address, int write_access);
795#else
796static inline int handle_mm_fault(struct mm_struct *mm,
797 struct vm_area_struct *vma, unsigned long address,
798 int write_access)
799{
800
801 BUG();
802 return VM_FAULT_SIGBUS;
803}
804#endif
805
806extern int make_pages_present(unsigned long addr, unsigned long end);
807extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
808
809int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start,
810 int len, int write, int force, struct page **pages, struct vm_area_struct **vmas);
811void print_bad_pte(struct vm_area_struct *, pte_t, unsigned long);
812
813extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
814extern void do_invalidatepage(struct page *page, unsigned long offset);
815
816int __set_page_dirty_nobuffers(struct page *page);
817int __set_page_dirty_no_writeback(struct page *page);
818int redirty_page_for_writepage(struct writeback_control *wbc,
819 struct page *page);
820int set_page_dirty(struct page *page);
821int set_page_dirty_lock(struct page *page);
822int clear_page_dirty_for_io(struct page *page);
823
824extern unsigned long move_page_tables(struct vm_area_struct *vma,
825 unsigned long old_addr, struct vm_area_struct *new_vma,
826 unsigned long new_addr, unsigned long len);
827extern unsigned long do_mremap(unsigned long addr,
828 unsigned long old_len, unsigned long new_len,
829 unsigned long flags, unsigned long new_addr);
830extern int mprotect_fixup(struct vm_area_struct *vma,
831 struct vm_area_struct **pprev, unsigned long start,
832 unsigned long end, unsigned long newflags);
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849struct shrinker {
850 int (*shrink)(int nr_to_scan, gfp_t gfp_mask);
851 int seeks;
852
853
854 struct list_head list;
855 long nr;
856};
857#define DEFAULT_SEEKS 2
858extern void register_shrinker(struct shrinker *);
859extern void unregister_shrinker(struct shrinker *);
860
861int vma_wants_writenotify(struct vm_area_struct *vma);
862
863extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
864
865#ifdef __PAGETABLE_PUD_FOLDED
866static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
867 unsigned long address)
868{
869 return 0;
870}
871#else
872int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
873#endif
874
875#ifdef __PAGETABLE_PMD_FOLDED
876static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
877 unsigned long address)
878{
879 return 0;
880}
881#else
882int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
883#endif
884
885int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
886int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
887
888
889
890
891
892#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
893static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
894{
895 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
896 NULL: pud_offset(pgd, address);
897}
898
899static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
900{
901 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
902 NULL: pmd_offset(pud, address);
903}
904#endif
905
906#if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS
907
908
909
910
911
912
913#define __pte_lockptr(page) &((page)->ptl)
914#define pte_lock_init(_page) do { \
915 spin_lock_init(__pte_lockptr(_page)); \
916} while (0)
917#define pte_lock_deinit(page) ((page)->mapping = NULL)
918#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
919#else
920
921
922
923#define pte_lock_init(page) do {} while (0)
924#define pte_lock_deinit(page) do {} while (0)
925#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
926#endif
927
928static inline void pgtable_page_ctor(struct page *page)
929{
930 pte_lock_init(page);
931 inc_zone_page_state(page, NR_PAGETABLE);
932}
933
934static inline void pgtable_page_dtor(struct page *page)
935{
936 pte_lock_deinit(page);
937 dec_zone_page_state(page, NR_PAGETABLE);
938}
939
940#define pte_offset_map_lock(mm, pmd, address, ptlp) \
941({ \
942 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
943 pte_t *__pte = pte_offset_map(pmd, address); \
944 *(ptlp) = __ptl; \
945 spin_lock(__ptl); \
946 __pte; \
947})
948
949#define pte_unmap_unlock(pte, ptl) do { \
950 spin_unlock(ptl); \
951 pte_unmap(pte); \
952} while (0)
953
954#define pte_alloc_map(mm, pmd, address) \
955 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
956 NULL: pte_offset_map(pmd, address))
957
958#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
959 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
960 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
961
962#define pte_alloc_kernel(pmd, address) \
963 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
964 NULL: pte_offset_kernel(pmd, address))
965
966extern void free_area_init(unsigned long * zones_size);
967extern void free_area_init_node(int nid, pg_data_t *pgdat,
968 unsigned long * zones_size, unsigned long zone_start_pfn,
969 unsigned long *zholes_size);
970#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998extern void free_area_init_nodes(unsigned long *max_zone_pfn);
999extern void add_active_range(unsigned int nid, unsigned long start_pfn,
1000 unsigned long end_pfn);
1001extern void shrink_active_range(unsigned int nid, unsigned long old_end_pfn,
1002 unsigned long new_end_pfn);
1003extern void push_node_boundaries(unsigned int nid, unsigned long start_pfn,
1004 unsigned long end_pfn);
1005extern void remove_all_active_ranges(void);
1006extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1007 unsigned long end_pfn);
1008extern void get_pfn_range_for_nid(unsigned int nid,
1009 unsigned long *start_pfn, unsigned long *end_pfn);
1010extern unsigned long find_min_pfn_with_active_regions(void);
1011extern unsigned long find_max_pfn_with_active_regions(void);
1012extern void free_bootmem_with_active_regions(int nid,
1013 unsigned long max_low_pfn);
1014extern void sparse_memory_present_with_active_regions(int nid);
1015#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1016extern int early_pfn_to_nid(unsigned long pfn);
1017#endif
1018#endif
1019extern void set_dma_reserve(unsigned long new_dma_reserve);
1020extern void memmap_init_zone(unsigned long, int, unsigned long,
1021 unsigned long, enum memmap_context);
1022extern void setup_per_zone_pages_min(void);
1023extern void mem_init(void);
1024extern void show_mem(void);
1025extern void si_meminfo(struct sysinfo * val);
1026extern void si_meminfo_node(struct sysinfo *val, int nid);
1027
1028#ifdef CONFIG_NUMA
1029extern void setup_per_cpu_pageset(void);
1030#else
1031static inline void setup_per_cpu_pageset(void) {}
1032#endif
1033
1034
1035void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
1036void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
1037void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *);
1038struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma,
1039 struct prio_tree_iter *iter);
1040
1041#define vma_prio_tree_foreach(vma, iter, root, begin, end) \
1042 for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \
1043 (vma = vma_prio_tree_next(vma, iter)); )
1044
1045static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1046 struct list_head *list)
1047{
1048 vma->shared.vm_set.parent = NULL;
1049 list_add_tail(&vma->shared.vm_set.list, list);
1050}
1051
1052
1053extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1054extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
1055 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1056extern struct vm_area_struct *vma_merge(struct mm_struct *,
1057 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1058 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1059 struct mempolicy *);
1060extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1061extern int split_vma(struct mm_struct *,
1062 struct vm_area_struct *, unsigned long addr, int new_below);
1063extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1064extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1065 struct rb_node **, struct rb_node *);
1066extern void unlink_file_vma(struct vm_area_struct *);
1067extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1068 unsigned long addr, unsigned long len, pgoff_t pgoff);
1069extern void exit_mmap(struct mm_struct *);
1070
1071#ifdef CONFIG_PROC_FS
1072
1073extern void added_exe_file_vma(struct mm_struct *mm);
1074extern void removed_exe_file_vma(struct mm_struct *mm);
1075#else
1076static inline void added_exe_file_vma(struct mm_struct *mm)
1077{}
1078
1079static inline void removed_exe_file_vma(struct mm_struct *mm)
1080{}
1081#endif
1082
1083extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1084extern int install_special_mapping(struct mm_struct *mm,
1085 unsigned long addr, unsigned long len,
1086 unsigned long flags, struct page **pages);
1087
1088extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1089
1090extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1091 unsigned long len, unsigned long prot,
1092 unsigned long flag, unsigned long pgoff);
1093extern unsigned long mmap_region(struct file *file, unsigned long addr,
1094 unsigned long len, unsigned long flags,
1095 unsigned int vm_flags, unsigned long pgoff,
1096 int accountable);
1097
1098static inline unsigned long do_mmap(struct file *file, unsigned long addr,
1099 unsigned long len, unsigned long prot,
1100 unsigned long flag, unsigned long offset)
1101{
1102 unsigned long ret = -EINVAL;
1103 if ((offset + PAGE_ALIGN(len)) < offset)
1104 goto out;
1105 if (!(offset & ~PAGE_MASK))
1106 ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1107out:
1108 return ret;
1109}
1110
1111extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1112
1113extern unsigned long do_brk(unsigned long, unsigned long);
1114
1115
1116extern unsigned long page_unuse(struct page *);
1117extern void truncate_inode_pages(struct address_space *, loff_t);
1118extern void truncate_inode_pages_range(struct address_space *,
1119 loff_t lstart, loff_t lend);
1120
1121
1122extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1123
1124
1125int write_one_page(struct page *page, int wait);
1126
1127
1128#define VM_MAX_READAHEAD 128
1129#define VM_MIN_READAHEAD 16
1130
1131int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
1132 pgoff_t offset, unsigned long nr_to_read);
1133int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1134 pgoff_t offset, unsigned long nr_to_read);
1135
1136void page_cache_sync_readahead(struct address_space *mapping,
1137 struct file_ra_state *ra,
1138 struct file *filp,
1139 pgoff_t offset,
1140 unsigned long size);
1141
1142void page_cache_async_readahead(struct address_space *mapping,
1143 struct file_ra_state *ra,
1144 struct file *filp,
1145 struct page *pg,
1146 pgoff_t offset,
1147 unsigned long size);
1148
1149unsigned long max_sane_readahead(unsigned long nr);
1150
1151
1152extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1153#ifdef CONFIG_IA64
1154extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1155#endif
1156extern int expand_stack_downwards(struct vm_area_struct *vma,
1157 unsigned long address);
1158
1159
1160extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1161extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1162 struct vm_area_struct **pprev);
1163
1164
1165
1166static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1167{
1168 struct vm_area_struct * vma = find_vma(mm,start_addr);
1169
1170 if (vma && end_addr <= vma->vm_start)
1171 vma = NULL;
1172 return vma;
1173}
1174
1175static inline unsigned long vma_pages(struct vm_area_struct *vma)
1176{
1177 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1178}
1179
1180pgprot_t vm_get_page_prot(unsigned long vm_flags);
1181struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
1182int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1183 unsigned long pfn, unsigned long size, pgprot_t);
1184int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
1185int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1186 unsigned long pfn);
1187int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1188 unsigned long pfn);
1189
1190struct page *follow_page(struct vm_area_struct *, unsigned long address,
1191 unsigned int foll_flags);
1192#define FOLL_WRITE 0x01
1193#define FOLL_TOUCH 0x02
1194#define FOLL_GET 0x04
1195#define FOLL_ANON 0x08
1196
1197typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1198 void *data);
1199extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1200 unsigned long size, pte_fn_t fn, void *data);
1201
1202#ifdef CONFIG_PROC_FS
1203void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
1204#else
1205static inline void vm_stat_account(struct mm_struct *mm,
1206 unsigned long flags, struct file *file, long pages)
1207{
1208}
1209#endif
1210
1211#ifdef CONFIG_DEBUG_PAGEALLOC
1212extern int debug_pagealloc_enabled;
1213
1214extern void kernel_map_pages(struct page *page, int numpages, int enable);
1215
1216static inline void enable_debug_pagealloc(void)
1217{
1218 debug_pagealloc_enabled = 1;
1219}
1220#ifdef CONFIG_HIBERNATION
1221extern bool kernel_page_present(struct page *page);
1222#endif
1223#else
1224static inline void
1225kernel_map_pages(struct page *page, int numpages, int enable) {}
1226static inline void enable_debug_pagealloc(void)
1227{
1228}
1229#ifdef CONFIG_HIBERNATION
1230static inline bool kernel_page_present(struct page *page) { return true; }
1231#endif
1232#endif
1233
1234extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk);
1235#ifdef __HAVE_ARCH_GATE_AREA
1236int in_gate_area_no_task(unsigned long addr);
1237int in_gate_area(struct task_struct *task, unsigned long addr);
1238#else
1239int in_gate_area_no_task(unsigned long addr);
1240#define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);})
1241#endif
1242
1243int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *,
1244 void __user *, size_t *, loff_t *);
1245unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
1246 unsigned long lru_pages);
1247
1248#ifndef CONFIG_MMU
1249#define randomize_va_space 0
1250#else
1251extern int randomize_va_space;
1252#endif
1253
1254const char * arch_vma_name(struct vm_area_struct *vma);
1255void print_vma_addr(char *prefix, unsigned long rip);
1256
1257struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
1258pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
1259pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
1260pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
1261pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
1262void *vmemmap_alloc_block(unsigned long size, int node);
1263void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
1264int vmemmap_populate_basepages(struct page *start_page,
1265 unsigned long pages, int node);
1266int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
1267void vmemmap_populate_print_last(void);
1268
1269#endif
1270#endif
1271