1#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
4#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
8#include <linux/gfp.h>
9#include <linux/list.h>
10#include <linux/mmdebug.h>
11#include <linux/mmzone.h>
12#include <linux/rbtree.h>
13#include <linux/prio_tree.h>
14#include <linux/debug_locks.h>
15#include <linux/mm_types.h>
16
17struct mempolicy;
18struct anon_vma;
19struct file_ra_state;
20struct user_struct;
21struct writeback_control;
22
23#ifndef CONFIG_DISCONTIGMEM
24extern unsigned long max_mapnr;
25#endif
26
27extern unsigned long num_physpages;
28extern void * high_memory;
29extern int page_cluster;
30
31#ifdef CONFIG_SYSCTL
32extern int sysctl_legacy_va_layout;
33#else
34#define sysctl_legacy_va_layout 0
35#endif
36
37extern unsigned long mmap_min_addr;
38
39#include <asm/page.h>
40#include <asm/pgtable.h>
41#include <asm/processor.h>
42
43#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
44
45
46#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
47
48
49
50
51
52
53
54
55
56
57extern struct kmem_cache *vm_area_cachep;
58
59
60
61
62
63
64struct vm_list_struct {
65 struct vm_list_struct *next;
66 struct vm_area_struct *vma;
67};
68
69#ifndef CONFIG_MMU
70extern struct rb_root nommu_vma_tree;
71extern struct rw_semaphore nommu_vma_sem;
72
73extern unsigned int kobjsize(const void *objp);
74#endif
75
76
77
78
79#define VM_READ 0x00000001
80#define VM_WRITE 0x00000002
81#define VM_EXEC 0x00000004
82#define VM_SHARED 0x00000008
83
84
85#define VM_MAYREAD 0x00000010
86#define VM_MAYWRITE 0x00000020
87#define VM_MAYEXEC 0x00000040
88#define VM_MAYSHARE 0x00000080
89
90#define VM_GROWSDOWN 0x00000100
91#define VM_GROWSUP 0x00000200
92#define VM_PFNMAP 0x00000400
93#define VM_DENYWRITE 0x00000800
94
95#define VM_EXECUTABLE 0x00001000
96#define VM_LOCKED 0x00002000
97#define VM_IO 0x00004000
98
99
100#define VM_SEQ_READ 0x00008000
101#define VM_RAND_READ 0x00010000
102
103#define VM_DONTCOPY 0x00020000
104#define VM_DONTEXPAND 0x00040000
105#define VM_RESERVED 0x00080000
106#define VM_ACCOUNT 0x00100000
107#define VM_NORESERVE 0x00200000
108#define VM_HUGETLB 0x00400000
109#define VM_NONLINEAR 0x00800000
110#define VM_MAPPED_COPY 0x01000000
111#define VM_INSERTPAGE 0x02000000
112#define VM_ALWAYSDUMP 0x04000000
113
114#define VM_CAN_NONLINEAR 0x08000000
115#define VM_MIXEDMAP 0x10000000
116#define VM_SAO 0x20000000
117
118#ifndef VM_STACK_DEFAULT_FLAGS
119#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
120#endif
121
122#ifdef CONFIG_STACK_GROWSUP
123#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
124#else
125#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
126#endif
127
128#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
129#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
130#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
131#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
132#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
133
134
135
136
137#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_RESERVED | VM_PFNMAP)
138
139
140
141
142
143extern pgprot_t protection_map[16];
144
145#define FAULT_FLAG_WRITE 0x01
146#define FAULT_FLAG_NONLINEAR 0x02
147
148
149
150
151
152
153
154
155
156
157
158struct vm_fault {
159 unsigned int flags;
160 pgoff_t pgoff;
161 void __user *virtual_address;
162
163 struct page *page;
164
165
166
167
168};
169
170
171
172
173
174
175struct vm_operations_struct {
176 void (*open)(struct vm_area_struct * area);
177 void (*close)(struct vm_area_struct * area);
178 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
179
180
181
182 int (*page_mkwrite)(struct vm_area_struct *vma, struct page *page);
183
184
185
186
187 int (*access)(struct vm_area_struct *vma, unsigned long addr,
188 void *buf, int len, int write);
189#ifdef CONFIG_NUMA
190
191
192
193
194
195
196
197 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
198
199
200
201
202
203
204
205
206
207
208
209 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
210 unsigned long addr);
211 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
212 const nodemask_t *to, unsigned long flags);
213#endif
214};
215
216struct mmu_gather;
217struct inode;
218
219#define page_private(page) ((page)->private)
220#define set_page_private(page, v) ((page)->private = (v))
221
222
223
224
225
226#include <linux/page-flags.h>
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244static inline int put_page_testzero(struct page *page)
245{
246 VM_BUG_ON(atomic_read(&page->_count) == 0);
247 return atomic_dec_and_test(&page->_count);
248}
249
250
251
252
253
254static inline int get_page_unless_zero(struct page *page)
255{
256 return atomic_inc_not_zero(&page->_count);
257}
258
259
260struct page *vmalloc_to_page(const void *addr);
261unsigned long vmalloc_to_pfn(const void *addr);
262
263
264
265
266
267
268
269static inline int is_vmalloc_addr(const void *x)
270{
271#ifdef CONFIG_MMU
272 unsigned long addr = (unsigned long)x;
273
274 return addr >= VMALLOC_START && addr < VMALLOC_END;
275#else
276 return 0;
277#endif
278}
279
280static inline struct page *compound_head(struct page *page)
281{
282 if (unlikely(PageTail(page)))
283 return page->first_page;
284 return page;
285}
286
287static inline int page_count(struct page *page)
288{
289 return atomic_read(&compound_head(page)->_count);
290}
291
292static inline void get_page(struct page *page)
293{
294 page = compound_head(page);
295 VM_BUG_ON(atomic_read(&page->_count) == 0);
296 atomic_inc(&page->_count);
297}
298
299static inline struct page *virt_to_head_page(const void *x)
300{
301 struct page *page = virt_to_page(x);
302 return compound_head(page);
303}
304
305
306
307
308
309static inline void init_page_count(struct page *page)
310{
311 atomic_set(&page->_count, 1);
312}
313
314void put_page(struct page *page);
315void put_pages_list(struct list_head *pages);
316
317void split_page(struct page *page, unsigned int order);
318
319
320
321
322
323
324typedef void compound_page_dtor(struct page *);
325
326static inline void set_compound_page_dtor(struct page *page,
327 compound_page_dtor *dtor)
328{
329 page[1].lru.next = (void *)dtor;
330}
331
332static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
333{
334 return (compound_page_dtor *)page[1].lru.next;
335}
336
337static inline int compound_order(struct page *page)
338{
339 if (!PageHead(page))
340 return 0;
341 return (unsigned long)page[1].lru.prev;
342}
343
344static inline void set_compound_order(struct page *page, unsigned long order)
345{
346 page[1].lru.prev = (void *)order;
347}
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
430#define SECTIONS_WIDTH SECTIONS_SHIFT
431#else
432#define SECTIONS_WIDTH 0
433#endif
434
435#define ZONES_WIDTH ZONES_SHIFT
436
437#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
438#define NODES_WIDTH NODES_SHIFT
439#else
440#ifdef CONFIG_SPARSEMEM_VMEMMAP
441#error "Vmemmap: No space for nodes field in page flags"
442#endif
443#define NODES_WIDTH 0
444#endif
445
446
447#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
448#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
449#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
450
451
452
453
454
455#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
456#define NODE_NOT_IN_PAGE_FLAGS
457#endif
458
459#ifndef PFN_SECTION_SHIFT
460#define PFN_SECTION_SHIFT 0
461#endif
462
463
464
465
466
467
468#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
469#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
470#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
471
472
473#ifdef NODE_NOT_IN_PAGEFLAGS
474#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
475#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
476 SECTIONS_PGOFF : ZONES_PGOFF)
477#else
478#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
479#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
480 NODES_PGOFF : ZONES_PGOFF)
481#endif
482
483#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
484
485#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
486#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
487#endif
488
489#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
490#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
491#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
492#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
493
494static inline enum zone_type page_zonenum(struct page *page)
495{
496 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
497}
498
499
500
501
502
503
504
505
506
507static inline int page_zone_id(struct page *page)
508{
509 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
510}
511
512static inline int zone_to_nid(struct zone *zone)
513{
514#ifdef CONFIG_NUMA
515 return zone->node;
516#else
517 return 0;
518#endif
519}
520
521#ifdef NODE_NOT_IN_PAGE_FLAGS
522extern int page_to_nid(struct page *page);
523#else
524static inline int page_to_nid(struct page *page)
525{
526 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
527}
528#endif
529
530static inline struct zone *page_zone(struct page *page)
531{
532 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
533}
534
535#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
536static inline unsigned long page_to_section(struct page *page)
537{
538 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
539}
540#endif
541
542static inline void set_page_zone(struct page *page, enum zone_type zone)
543{
544 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
545 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
546}
547
548static inline void set_page_node(struct page *page, unsigned long node)
549{
550 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
551 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
552}
553
554static inline void set_page_section(struct page *page, unsigned long section)
555{
556 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
557 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
558}
559
560static inline void set_page_links(struct page *page, enum zone_type zone,
561 unsigned long node, unsigned long pfn)
562{
563 set_page_zone(page, zone);
564 set_page_node(page, node);
565 set_page_section(page, pfn_to_section_nr(pfn));
566}
567
568
569
570
571
572static inline unsigned long round_hint_to_min(unsigned long hint)
573{
574#ifdef CONFIG_SECURITY
575 hint &= PAGE_MASK;
576 if (((void *)hint != NULL) &&
577 (hint < mmap_min_addr))
578 return PAGE_ALIGN(mmap_min_addr);
579#endif
580 return hint;
581}
582
583
584
585
586#include <linux/vmstat.h>
587
588static __always_inline void *lowmem_page_address(struct page *page)
589{
590 return __va(page_to_pfn(page) << PAGE_SHIFT);
591}
592
593#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
594#define HASHED_PAGE_VIRTUAL
595#endif
596
597#if defined(WANT_PAGE_VIRTUAL)
598#define page_address(page) ((page)->virtual)
599#define set_page_address(page, address) \
600 do { \
601 (page)->virtual = (address); \
602 } while(0)
603#define page_address_init() do { } while(0)
604#endif
605
606#if defined(HASHED_PAGE_VIRTUAL)
607void *page_address(struct page *page);
608void set_page_address(struct page *page, void *virtual);
609void page_address_init(void);
610#endif
611
612#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
613#define page_address(page) lowmem_page_address(page)
614#define set_page_address(page, address) do { } while(0)
615#define page_address_init() do { } while(0)
616#endif
617
618
619
620
621
622
623
624
625
626
627#define PAGE_MAPPING_ANON 1
628
629extern struct address_space swapper_space;
630static inline struct address_space *page_mapping(struct page *page)
631{
632 struct address_space *mapping = page->mapping;
633
634 VM_BUG_ON(PageSlab(page));
635#ifdef CONFIG_SWAP
636 if (unlikely(PageSwapCache(page)))
637 mapping = &swapper_space;
638 else
639#endif
640 if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
641 mapping = NULL;
642 return mapping;
643}
644
645static inline int PageAnon(struct page *page)
646{
647 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
648}
649
650
651
652
653
654static inline pgoff_t page_index(struct page *page)
655{
656 if (unlikely(PageSwapCache(page)))
657 return page_private(page);
658 return page->index;
659}
660
661
662
663
664
665
666static inline void reset_page_mapcount(struct page *page)
667{
668 atomic_set(&(page)->_mapcount, -1);
669}
670
671static inline int page_mapcount(struct page *page)
672{
673 return atomic_read(&(page)->_mapcount) + 1;
674}
675
676
677
678
679static inline int page_mapped(struct page *page)
680{
681 return atomic_read(&(page)->_mapcount) >= 0;
682}
683
684
685
686
687
688
689
690#define VM_FAULT_MINOR 0
691
692#define VM_FAULT_OOM 0x0001
693#define VM_FAULT_SIGBUS 0x0002
694#define VM_FAULT_MAJOR 0x0004
695#define VM_FAULT_WRITE 0x0008
696
697#define VM_FAULT_NOPAGE 0x0100
698#define VM_FAULT_LOCKED 0x0200
699
700#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS)
701
702#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
703
704extern void show_free_areas(void);
705
706#ifdef CONFIG_SHMEM
707extern int shmem_lock(struct file *file, int lock, struct user_struct *user);
708#else
709static inline int shmem_lock(struct file *file, int lock,
710 struct user_struct *user)
711{
712 return 0;
713}
714#endif
715struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags);
716
717int shmem_zero_setup(struct vm_area_struct *);
718
719#ifndef CONFIG_MMU
720extern unsigned long shmem_get_unmapped_area(struct file *file,
721 unsigned long addr,
722 unsigned long len,
723 unsigned long pgoff,
724 unsigned long flags);
725#endif
726
727extern int can_do_mlock(void);
728extern int user_shm_lock(size_t, struct user_struct *);
729extern void user_shm_unlock(size_t, struct user_struct *);
730
731
732
733
734struct zap_details {
735 struct vm_area_struct *nonlinear_vma;
736 struct address_space *check_mapping;
737 pgoff_t first_index;
738 pgoff_t last_index;
739 spinlock_t *i_mmap_lock;
740 unsigned long truncate_count;
741};
742
743struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
744 pte_t pte);
745
746int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
747 unsigned long size);
748unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
749 unsigned long size, struct zap_details *);
750unsigned long unmap_vmas(struct mmu_gather **tlb,
751 struct vm_area_struct *start_vma, unsigned long start_addr,
752 unsigned long end_addr, unsigned long *nr_accounted,
753 struct zap_details *);
754
755
756
757
758
759
760
761
762
763
764
765struct mm_walk {
766 int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
767 int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
768 int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
769 int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
770 int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
771 struct mm_struct *mm;
772 void *private;
773};
774
775int walk_page_range(unsigned long addr, unsigned long end,
776 struct mm_walk *walk);
777void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
778 unsigned long end, unsigned long floor, unsigned long ceiling);
779int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
780 struct vm_area_struct *vma);
781void unmap_mapping_range(struct address_space *mapping,
782 loff_t const holebegin, loff_t const holelen, int even_cows);
783int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
784 void *buf, int len, int write);
785
786static inline void unmap_shared_mapping_range(struct address_space *mapping,
787 loff_t const holebegin, loff_t const holelen)
788{
789 unmap_mapping_range(mapping, holebegin, holelen, 0);
790}
791
792extern int vmtruncate(struct inode * inode, loff_t offset);
793extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end);
794
795#ifdef CONFIG_MMU
796extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
797 unsigned long address, int write_access);
798#else
799static inline int handle_mm_fault(struct mm_struct *mm,
800 struct vm_area_struct *vma, unsigned long address,
801 int write_access)
802{
803
804 BUG();
805 return VM_FAULT_SIGBUS;
806}
807#endif
808
809extern int make_pages_present(unsigned long addr, unsigned long end);
810extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
811
812int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start,
813 int len, int write, int force, struct page **pages, struct vm_area_struct **vmas);
814
815extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
816extern void do_invalidatepage(struct page *page, unsigned long offset);
817
818int __set_page_dirty_nobuffers(struct page *page);
819int __set_page_dirty_no_writeback(struct page *page);
820int redirty_page_for_writepage(struct writeback_control *wbc,
821 struct page *page);
822int set_page_dirty(struct page *page);
823int set_page_dirty_lock(struct page *page);
824int clear_page_dirty_for_io(struct page *page);
825
826extern unsigned long move_page_tables(struct vm_area_struct *vma,
827 unsigned long old_addr, struct vm_area_struct *new_vma,
828 unsigned long new_addr, unsigned long len);
829extern unsigned long do_mremap(unsigned long addr,
830 unsigned long old_len, unsigned long new_len,
831 unsigned long flags, unsigned long new_addr);
832extern int mprotect_fixup(struct vm_area_struct *vma,
833 struct vm_area_struct **pprev, unsigned long start,
834 unsigned long end, unsigned long newflags);
835
836
837
838
839
840
841
842
843
844
845
846int get_user_pages_fast(unsigned long start, int nr_pages, int write,
847 struct page **pages);
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864struct shrinker {
865 int (*shrink)(int nr_to_scan, gfp_t gfp_mask);
866 int seeks;
867
868
869 struct list_head list;
870 long nr;
871};
872#define DEFAULT_SEEKS 2
873extern void register_shrinker(struct shrinker *);
874extern void unregister_shrinker(struct shrinker *);
875
876int vma_wants_writenotify(struct vm_area_struct *vma);
877
878extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
879
880#ifdef __PAGETABLE_PUD_FOLDED
881static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
882 unsigned long address)
883{
884 return 0;
885}
886#else
887int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
888#endif
889
890#ifdef __PAGETABLE_PMD_FOLDED
891static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
892 unsigned long address)
893{
894 return 0;
895}
896#else
897int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
898#endif
899
900int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
901int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
902
903
904
905
906
907#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
908static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
909{
910 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
911 NULL: pud_offset(pgd, address);
912}
913
914static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
915{
916 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
917 NULL: pmd_offset(pud, address);
918}
919#endif
920
921#if USE_SPLIT_PTLOCKS
922
923
924
925
926
927
928#define __pte_lockptr(page) &((page)->ptl)
929#define pte_lock_init(_page) do { \
930 spin_lock_init(__pte_lockptr(_page)); \
931} while (0)
932#define pte_lock_deinit(page) ((page)->mapping = NULL)
933#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
934#else
935
936
937
938#define pte_lock_init(page) do {} while (0)
939#define pte_lock_deinit(page) do {} while (0)
940#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
941#endif
942
943static inline void pgtable_page_ctor(struct page *page)
944{
945 pte_lock_init(page);
946 inc_zone_page_state(page, NR_PAGETABLE);
947}
948
949static inline void pgtable_page_dtor(struct page *page)
950{
951 pte_lock_deinit(page);
952 dec_zone_page_state(page, NR_PAGETABLE);
953}
954
955#define pte_offset_map_lock(mm, pmd, address, ptlp) \
956({ \
957 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
958 pte_t *__pte = pte_offset_map(pmd, address); \
959 *(ptlp) = __ptl; \
960 spin_lock(__ptl); \
961 __pte; \
962})
963
964#define pte_unmap_unlock(pte, ptl) do { \
965 spin_unlock(ptl); \
966 pte_unmap(pte); \
967} while (0)
968
969#define pte_alloc_map(mm, pmd, address) \
970 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
971 NULL: pte_offset_map(pmd, address))
972
973#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
974 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \
975 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
976
977#define pte_alloc_kernel(pmd, address) \
978 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
979 NULL: pte_offset_kernel(pmd, address))
980
981extern void free_area_init(unsigned long * zones_size);
982extern void free_area_init_node(int nid, unsigned long * zones_size,
983 unsigned long zone_start_pfn, unsigned long *zholes_size);
984#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012extern void free_area_init_nodes(unsigned long *max_zone_pfn);
1013extern void add_active_range(unsigned int nid, unsigned long start_pfn,
1014 unsigned long end_pfn);
1015extern void remove_active_range(unsigned int nid, unsigned long start_pfn,
1016 unsigned long end_pfn);
1017extern void push_node_boundaries(unsigned int nid, unsigned long start_pfn,
1018 unsigned long end_pfn);
1019extern void remove_all_active_ranges(void);
1020extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1021 unsigned long end_pfn);
1022extern void get_pfn_range_for_nid(unsigned int nid,
1023 unsigned long *start_pfn, unsigned long *end_pfn);
1024extern unsigned long find_min_pfn_with_active_regions(void);
1025extern void free_bootmem_with_active_regions(int nid,
1026 unsigned long max_low_pfn);
1027typedef int (*work_fn_t)(unsigned long, unsigned long, void *);
1028extern void work_with_active_regions(int nid, work_fn_t work_fn, void *data);
1029extern void sparse_memory_present_with_active_regions(int nid);
1030#endif
1031
1032#if !defined(CONFIG_ARCH_POPULATES_NODE_MAP) && \
1033 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1034static inline int __early_pfn_to_nid(unsigned long pfn)
1035{
1036 return 0;
1037}
1038#else
1039
1040extern int __meminit early_pfn_to_nid(unsigned long pfn);
1041#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1042
1043extern int __meminit __early_pfn_to_nid(unsigned long pfn);
1044#endif
1045#endif
1046
1047extern void set_dma_reserve(unsigned long new_dma_reserve);
1048extern void memmap_init_zone(unsigned long, int, unsigned long,
1049 unsigned long, enum memmap_context);
1050extern void setup_per_zone_pages_min(void);
1051extern void mem_init(void);
1052extern void show_mem(void);
1053extern void si_meminfo(struct sysinfo * val);
1054extern void si_meminfo_node(struct sysinfo *val, int nid);
1055extern int after_bootmem;
1056
1057#ifdef CONFIG_NUMA
1058extern void setup_per_cpu_pageset(void);
1059#else
1060static inline void setup_per_cpu_pageset(void) {}
1061#endif
1062
1063
1064void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
1065void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
1066void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *);
1067struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma,
1068 struct prio_tree_iter *iter);
1069
1070#define vma_prio_tree_foreach(vma, iter, root, begin, end) \
1071 for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \
1072 (vma = vma_prio_tree_next(vma, iter)); )
1073
1074static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1075 struct list_head *list)
1076{
1077 vma->shared.vm_set.parent = NULL;
1078 list_add_tail(&vma->shared.vm_set.list, list);
1079}
1080
1081
1082extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
1083extern void vma_adjust(struct vm_area_struct *vma, unsigned long start,
1084 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1085extern struct vm_area_struct *vma_merge(struct mm_struct *,
1086 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1087 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1088 struct mempolicy *);
1089extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1090extern int split_vma(struct mm_struct *,
1091 struct vm_area_struct *, unsigned long addr, int new_below);
1092extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1093extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1094 struct rb_node **, struct rb_node *);
1095extern void unlink_file_vma(struct vm_area_struct *);
1096extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
1097 unsigned long addr, unsigned long len, pgoff_t pgoff);
1098extern void exit_mmap(struct mm_struct *);
1099
1100extern int mm_take_all_locks(struct mm_struct *mm);
1101extern void mm_drop_all_locks(struct mm_struct *mm);
1102
1103#ifdef CONFIG_PROC_FS
1104
1105extern void added_exe_file_vma(struct mm_struct *mm);
1106extern void removed_exe_file_vma(struct mm_struct *mm);
1107#else
1108static inline void added_exe_file_vma(struct mm_struct *mm)
1109{}
1110
1111static inline void removed_exe_file_vma(struct mm_struct *mm)
1112{}
1113#endif
1114
1115extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
1116extern int install_special_mapping(struct mm_struct *mm,
1117 unsigned long addr, unsigned long len,
1118 unsigned long flags, struct page **pages);
1119
1120extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1121
1122extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1123 unsigned long len, unsigned long prot,
1124 unsigned long flag, unsigned long pgoff);
1125extern unsigned long mmap_region(struct file *file, unsigned long addr,
1126 unsigned long len, unsigned long flags,
1127 unsigned int vm_flags, unsigned long pgoff,
1128 int accountable);
1129
1130static inline unsigned long do_mmap(struct file *file, unsigned long addr,
1131 unsigned long len, unsigned long prot,
1132 unsigned long flag, unsigned long offset)
1133{
1134 unsigned long ret = -EINVAL;
1135 if ((offset + PAGE_ALIGN(len)) < offset)
1136 goto out;
1137 if (!(offset & ~PAGE_MASK))
1138 ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
1139out:
1140 return ret;
1141}
1142
1143extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1144
1145extern unsigned long do_brk(unsigned long, unsigned long);
1146
1147
1148extern unsigned long page_unuse(struct page *);
1149extern void truncate_inode_pages(struct address_space *, loff_t);
1150extern void truncate_inode_pages_range(struct address_space *,
1151 loff_t lstart, loff_t lend);
1152
1153
1154extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
1155
1156
1157int write_one_page(struct page *page, int wait);
1158
1159
1160#define VM_MAX_READAHEAD 128
1161#define VM_MIN_READAHEAD 16
1162
1163int do_page_cache_readahead(struct address_space *mapping, struct file *filp,
1164 pgoff_t offset, unsigned long nr_to_read);
1165int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
1166 pgoff_t offset, unsigned long nr_to_read);
1167
1168void page_cache_sync_readahead(struct address_space *mapping,
1169 struct file_ra_state *ra,
1170 struct file *filp,
1171 pgoff_t offset,
1172 unsigned long size);
1173
1174void page_cache_async_readahead(struct address_space *mapping,
1175 struct file_ra_state *ra,
1176 struct file *filp,
1177 struct page *pg,
1178 pgoff_t offset,
1179 unsigned long size);
1180
1181unsigned long max_sane_readahead(unsigned long nr);
1182
1183
1184extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
1185#ifdef CONFIG_IA64
1186extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
1187#endif
1188extern int expand_stack_downwards(struct vm_area_struct *vma,
1189 unsigned long address);
1190
1191
1192extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1193extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1194 struct vm_area_struct **pprev);
1195
1196
1197
1198static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1199{
1200 struct vm_area_struct * vma = find_vma(mm,start_addr);
1201
1202 if (vma && end_addr <= vma->vm_start)
1203 vma = NULL;
1204 return vma;
1205}
1206
1207static inline unsigned long vma_pages(struct vm_area_struct *vma)
1208{
1209 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1210}
1211
1212pgprot_t vm_get_page_prot(unsigned long vm_flags);
1213struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
1214int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1215 unsigned long pfn, unsigned long size, pgprot_t);
1216int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
1217int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1218 unsigned long pfn);
1219int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1220 unsigned long pfn);
1221
1222struct page *follow_page(struct vm_area_struct *, unsigned long address,
1223 unsigned int foll_flags);
1224#define FOLL_WRITE 0x01
1225#define FOLL_TOUCH 0x02
1226#define FOLL_GET 0x04
1227#define FOLL_ANON 0x08
1228
1229typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1230 void *data);
1231extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1232 unsigned long size, pte_fn_t fn, void *data);
1233
1234#ifdef CONFIG_PROC_FS
1235void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
1236#else
1237static inline void vm_stat_account(struct mm_struct *mm,
1238 unsigned long flags, struct file *file, long pages)
1239{
1240}
1241#endif
1242
1243#ifdef CONFIG_DEBUG_PAGEALLOC
1244extern int debug_pagealloc_enabled;
1245
1246extern void kernel_map_pages(struct page *page, int numpages, int enable);
1247
1248static inline void enable_debug_pagealloc(void)
1249{
1250 debug_pagealloc_enabled = 1;
1251}
1252#ifdef CONFIG_HIBERNATION
1253extern bool kernel_page_present(struct page *page);
1254#endif
1255#else
1256static inline void
1257kernel_map_pages(struct page *page, int numpages, int enable) {}
1258static inline void enable_debug_pagealloc(void)
1259{
1260}
1261#ifdef CONFIG_HIBERNATION
1262static inline bool kernel_page_present(struct page *page) { return true; }
1263#endif
1264#endif
1265
1266extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk);
1267#ifdef __HAVE_ARCH_GATE_AREA
1268int in_gate_area_no_task(unsigned long addr);
1269int in_gate_area(struct task_struct *task, unsigned long addr);
1270#else
1271int in_gate_area_no_task(unsigned long addr);
1272#define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);})
1273#endif
1274
1275int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *,
1276 void __user *, size_t *, loff_t *);
1277unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
1278 unsigned long lru_pages);
1279
1280#ifndef CONFIG_MMU
1281#define randomize_va_space 0
1282#else
1283extern int randomize_va_space;
1284#endif
1285
1286const char * arch_vma_name(struct vm_area_struct *vma);
1287void print_vma_addr(char *prefix, unsigned long rip);
1288
1289struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
1290pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
1291pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
1292pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
1293pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
1294void *vmemmap_alloc_block(unsigned long size, int node);
1295void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
1296int vmemmap_populate_basepages(struct page *start_page,
1297 unsigned long pages, int node);
1298int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
1299void vmemmap_populate_print_last(void);
1300
1301#endif
1302#endif
1303