1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64#ifndef _VM_VM_MAP_H_
65#define _VM_VM_MAP_H_
66
67#include <mach/mach_types.h>
68#include <mach/kern_return.h>
69#include <mach/boolean.h>
70#include <mach/vm_types.h>
71#include <mach/vm_prot.h>
72#include <mach/vm_inherit.h>
73#include <mach/vm_behavior.h>
74#include <mach/vm_param.h>
75#include <vm/pmap.h>
76
77#ifdef KERNEL_PRIVATE
78
79#include <sys/cdefs.h>
80
81__BEGIN_DECLS
82
83extern void vm_map_reference(vm_map_t map);
84extern vm_map_t current_map(void);
85
86__END_DECLS
87
88#ifdef MACH_KERNEL_PRIVATE
89
90#include <task_swapper.h>
91#include <mach_assert.h>
92
93#include <vm/vm_object.h>
94#include <vm/vm_page.h>
95#include <kern/lock.h>
96#include <kern/zalloc.h>
97#include <kern/macro_help.h>
98
99#include <kern/thread.h>
100
101#define current_map_fast() (current_thread()->map)
102#define current_map() (current_map_fast())
103
104
105
106
107
108
109
110
111
112
113typedef struct vm_map_entry *vm_map_entry_t;
114#define VM_MAP_ENTRY_NULL ((vm_map_entry_t) 0)
115
116
117
118
119
120
121
122
123
124typedef union vm_map_object {
125 vm_object_t vm_object;
126 vm_map_t sub_map;
127} vm_map_object_t;
128
129#define named_entry_lock_init(object) mutex_init(&(object)->Lock, 0)
130#define named_entry_lock(object) mutex_lock(&(object)->Lock)
131#define named_entry_unlock(object) mutex_unlock(&(object)->Lock)
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151struct vm_named_entry {
152 decl_mutex_data(, Lock)
153 union {
154 vm_object_t object;
155 memory_object_t pager;
156 vm_map_t map;
157 } backing;
158 vm_object_offset_t offset;
159 vm_object_size_t size;
160 vm_prot_t protection;
161 int ref_count;
162 unsigned int
163 internal:1,
164 is_sub_map:1,
165 is_pager:1;
166};
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181struct vm_map_links {
182 struct vm_map_entry *prev;
183 struct vm_map_entry *next;
184 vm_map_offset_t start;
185 vm_map_offset_t end;
186};
187
188struct vm_map_entry {
189 struct vm_map_links links;
190#define vme_prev links.prev
191#define vme_next links.next
192#define vme_start links.start
193#define vme_end links.end
194 union vm_map_object object;
195 vm_object_offset_t offset;
196 unsigned int
197 is_shared:1,
198 is_sub_map:1,
199 in_transition:1,
200 needs_wakeup:1,
201 behavior:2,
202
203 needs_copy:1,
204
205 protection:3,
206 max_protection:3,
207 inheritance:2,
208 use_pmap:1,
209 alias:8,
210 pad:8;
211 unsigned short wired_count;
212 unsigned short user_wired_count;
213};
214
215
216
217
218
219#define MAX_WIRE_COUNT 65535
220
221
222
223
224
225
226
227
228
229struct vm_map_header {
230 struct vm_map_links links;
231 int nentries;
232 boolean_t entries_pageable;
233
234};
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251struct vm_map {
252 lock_t lock;
253 struct vm_map_header hdr;
254#define min_offset hdr.links.start
255#define max_offset hdr.links.end
256 pmap_t pmap;
257 vm_map_size_t size;
258 int ref_count;
259#if TASK_SWAPPER
260 int res_count;
261 int sw_state;
262#endif
263 decl_mutex_data(, s_lock)
264 vm_map_entry_t hint;
265 vm_map_entry_t first_free;
266 boolean_t wait_for_space;
267
268 boolean_t wiring_required;
269 boolean_t no_zero_fill;
270 boolean_t mapped;
271 unsigned int timestamp;
272} ;
273
274#define vm_map_to_entry(map) ((struct vm_map_entry *) &(map)->hdr.links)
275#define vm_map_first_entry(map) ((map)->hdr.links.next)
276#define vm_map_last_entry(map) ((map)->hdr.links.prev)
277
278#if TASK_SWAPPER
279
280
281
282#define MAP_SW_IN 1
283#define MAP_SW_OUT 2
284#endif
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300typedef struct vm_map_version {
301 unsigned int main_timestamp;
302} vm_map_version_t;
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335struct vm_map_copy {
336 int type;
337#define VM_MAP_COPY_ENTRY_LIST 1
338#define VM_MAP_COPY_OBJECT 2
339#define VM_MAP_COPY_KERNEL_BUFFER 3
340 vm_object_offset_t offset;
341 vm_map_size_t size;
342 union {
343 struct vm_map_header hdr;
344 vm_object_t object;
345 struct {
346 void *kdata;
347 vm_size_t kalloc_size;
348 } c_k;
349 } c_u;
350};
351
352
353#define cpy_hdr c_u.hdr
354
355#define cpy_object c_u.object
356
357#define cpy_kdata c_u.c_k.kdata
358#define cpy_kalloc_size c_u.c_k.kalloc_size
359
360
361
362
363
364
365#define vm_map_copy_to_entry(copy) \
366 ((struct vm_map_entry *) &(copy)->cpy_hdr.links)
367#define vm_map_copy_first_entry(copy) \
368 ((copy)->cpy_hdr.links.next)
369#define vm_map_copy_last_entry(copy) \
370 ((copy)->cpy_hdr.links.prev)
371
372
373
374
375
376
377
378
379
380#define vm_map_lock_init(map) \
381 ((map)->timestamp = 0 , \
382 lock_init(&(map)->lock, TRUE, 0, 0))
383
384#define vm_map_lock(map) lock_write(&(map)->lock)
385#define vm_map_unlock(map) \
386 ((map)->timestamp++ , lock_write_done(&(map)->lock))
387#define vm_map_lock_read(map) lock_read(&(map)->lock)
388#define vm_map_unlock_read(map) lock_read_done(&(map)->lock)
389#define vm_map_lock_write_to_read(map) \
390 ((map)->timestamp++ , lock_write_to_read(&(map)->lock))
391#define vm_map_lock_read_to_write(map) lock_read_to_write(&(map)->lock)
392
393
394
395
396
397
398extern void vm_map_init(void);
399
400
401
402extern kern_return_t vm_map_find_space(
403 vm_map_t map,
404 vm_map_address_t *address,
405 vm_map_size_t size,
406 vm_map_offset_t mask,
407 vm_map_entry_t *o_entry);
408
409
410extern boolean_t vm_map_lookup_entry(
411 vm_map_t map,
412 vm_map_address_t address,
413 vm_map_entry_t *entry);
414
415
416
417extern kern_return_t vm_map_lookup_locked(
418 vm_map_t *var_map,
419 vm_map_address_t vaddr,
420 vm_prot_t fault_type,
421 vm_map_version_t *out_version,
422 vm_object_t *object,
423 vm_object_offset_t *offset,
424 vm_prot_t *out_prot,
425 boolean_t *wired,
426 int *behavior,
427 vm_map_offset_t *lo_offset,
428 vm_map_offset_t *hi_offset,
429 vm_map_t *real_map);
430
431
432extern boolean_t vm_map_verify(
433 vm_map_t map,
434 vm_map_version_t *version);
435
436extern vm_map_entry_t vm_map_entry_insert(
437 vm_map_t map,
438 vm_map_entry_t insp_entry,
439 vm_map_offset_t start,
440 vm_map_offset_t end,
441 vm_object_t object,
442 vm_object_offset_t offset,
443 boolean_t needs_copy,
444 boolean_t is_shared,
445 boolean_t in_transition,
446 vm_prot_t cur_protection,
447 vm_prot_t max_protection,
448 vm_behavior_t behavior,
449 vm_inherit_t inheritance,
450 unsigned wired_count);
451
452
453
454
455
456#define vm_map_min(map) ((map)->min_offset)
457
458
459
460#define vm_map_max(map) ((map)->max_offset)
461
462
463#define vm_map_pmap(map) ((map)->pmap)
464
465
466
467#define vm_map_verify_done(map, version) vm_map_unlock_read(map)
468
469
470
471
472
473
474
475#if TASK_SWAPPER
476
477#if MACH_ASSERT
478
479extern void vm_map_reference(
480 vm_map_t map);
481
482extern void vm_map_res_deallocate(
483 vm_map_t map);
484
485extern void vm_map_res_reference(
486 vm_map_t map);
487
488extern void vm_map_reference_swap(
489 vm_map_t map);
490
491#else
492
493#define vm_map_reference(map) \
494MACRO_BEGIN \
495 vm_map_t Map = (map); \
496 if (Map) { \
497 mutex_lock(&Map->s_lock); \
498 Map->res_count++; \
499 Map->ref_count++; \
500 mutex_unlock(&Map->s_lock); \
501 } \
502MACRO_END
503
504#define vm_map_res_reference(map) \
505MACRO_BEGIN \
506 vm_map_t Lmap = (map); \
507 if (Lmap->res_count == 0) { \
508 mutex_unlock(&Lmap->s_lock);\
509 vm_map_lock(Lmap); \
510 vm_map_swapin(Lmap); \
511 mutex_lock(&Lmap->s_lock); \
512 ++Lmap->res_count; \
513 vm_map_unlock(Lmap); \
514 } else \
515 ++Lmap->res_count; \
516MACRO_END
517
518#define vm_map_res_deallocate(map) \
519MACRO_BEGIN \
520 vm_map_t Map = (map); \
521 if (--Map->res_count == 0) { \
522 mutex_unlock(&Map->s_lock); \
523 vm_map_lock(Map); \
524 vm_map_swapout(Map); \
525 vm_map_unlock(Map); \
526 mutex_lock(&Map->s_lock); \
527 } \
528MACRO_END
529
530#define vm_map_reference_swap(map) \
531MACRO_BEGIN \
532 vm_map_t Map = (map); \
533 mutex_lock(&Map->s_lock); \
534 ++Map->ref_count; \
535 vm_map_res_reference(Map); \
536 mutex_unlock(&Map->s_lock); \
537MACRO_END
538#endif
539
540extern void vm_map_swapin(
541 vm_map_t map);
542
543extern void vm_map_swapout(
544 vm_map_t map);
545
546#else
547
548#define vm_map_reference(map) \
549MACRO_BEGIN \
550 vm_map_t Map = (map); \
551 if (Map) { \
552 mutex_lock(&Map->s_lock); \
553 Map->ref_count++; \
554 mutex_unlock(&Map->s_lock); \
555 } \
556MACRO_END
557
558#define vm_map_reference_swap(map) vm_map_reference(map)
559#define vm_map_res_reference(map)
560#define vm_map_res_deallocate(map)
561
562#endif
563
564
565
566
567
568extern vm_object_t vm_submap_object;
569
570
571
572
573#define vm_map_entry_wait(map, interruptible) \
574 ((map)->timestamp++ , \
575 thread_sleep_lock_write((event_t)&(map)->hdr, \
576 &(map)->lock, interruptible))
577
578
579#define vm_map_entry_wakeup(map) \
580 thread_wakeup((event_t)(&(map)->hdr))
581
582
583#define vm_map_ref_fast(map) \
584 MACRO_BEGIN \
585 mutex_lock(&map->s_lock); \
586 map->ref_count++; \
587 vm_map_res_reference(map); \
588 mutex_unlock(&map->s_lock); \
589 MACRO_END
590
591#define vm_map_dealloc_fast(map) \
592 MACRO_BEGIN \
593 register int c; \
594 \
595 mutex_lock(&map->s_lock); \
596 c = --map->ref_count; \
597 if (c > 0) \
598 vm_map_res_deallocate(map); \
599 mutex_unlock(&map->s_lock); \
600 if (c == 0) \
601 vm_map_destroy(map); \
602 MACRO_END
603
604
605
606extern void vm_map_simplify_entry(
607 vm_map_t map,
608 vm_map_entry_t this_entry);
609extern void vm_map_simplify(
610 vm_map_t map,
611 vm_map_offset_t start);
612
613
614extern vm_map_copy_t vm_map_copy_copy(
615 vm_map_copy_t copy);
616
617
618extern kern_return_t vm_map_copyin_object(
619 vm_object_t object,
620 vm_object_offset_t offset,
621 vm_object_size_t size,
622 vm_map_copy_t *copy_result);
623
624
625extern kern_return_t vm_map_enter(
626 vm_map_t map,
627 vm_map_offset_t *address,
628 vm_map_size_t size,
629 vm_map_offset_t mask,
630 int flags,
631 vm_object_t object,
632 vm_object_offset_t offset,
633 boolean_t needs_copy,
634 vm_prot_t cur_protection,
635 vm_prot_t max_protection,
636 vm_inherit_t inheritance);
637
638
639extern kern_return_t vm_map_enter_cpm(
640 vm_map_t map,
641 vm_map_address_t *addr,
642 vm_map_size_t size,
643 int flags);
644
645extern kern_return_t vm_map_remap(
646 vm_map_t target_map,
647 vm_map_offset_t *address,
648 vm_map_size_t size,
649 vm_map_offset_t mask,
650 boolean_t anywhere,
651 vm_map_t src_map,
652 vm_map_offset_t memory_address,
653 boolean_t copy,
654 vm_prot_t *cur_protection,
655 vm_prot_t *max_protection,
656 vm_inherit_t inheritance);
657
658
659
660
661
662extern kern_return_t vm_map_write_user(
663 vm_map_t map,
664 void *src_p,
665 vm_map_offset_t dst_addr,
666 vm_size_t size);
667
668extern kern_return_t vm_map_read_user(
669 vm_map_t map,
670 vm_map_offset_t src_addr,
671 void *dst_p,
672 vm_size_t size);
673
674
675extern vm_map_t vm_map_fork(
676 vm_map_t old_map);
677
678
679extern kern_return_t vm_map_inherit(
680 vm_map_t map,
681 vm_map_offset_t start,
682 vm_map_offset_t end,
683 vm_inherit_t new_inheritance);
684
685
686extern kern_return_t vm_map_machine_attribute(
687 vm_map_t map,
688 vm_map_offset_t start,
689 vm_map_offset_t end,
690 vm_machine_attribute_t attribute,
691 vm_machine_attribute_val_t* value);
692
693extern kern_return_t vm_map_msync(
694 vm_map_t map,
695 vm_map_address_t address,
696 vm_map_size_t size,
697 vm_sync_t sync_flags);
698
699
700extern kern_return_t vm_map_behavior_set(
701 vm_map_t map,
702 vm_map_offset_t start,
703 vm_map_offset_t end,
704 vm_behavior_t new_behavior);
705
706extern kern_return_t vm_map_purgable_control(
707 vm_map_t map,
708 vm_map_offset_t address,
709 vm_purgable_t control,
710 int *state);
711
712extern kern_return_t vm_map_region(
713 vm_map_t map,
714 vm_map_offset_t *address,
715 vm_map_size_t *size,
716 vm_region_flavor_t flavor,
717 vm_region_info_t info,
718 mach_msg_type_number_t *count,
719 mach_port_t *object_name);
720
721extern kern_return_t vm_map_region_recurse_64(
722 vm_map_t map,
723 vm_map_offset_t *address,
724 vm_map_size_t *size,
725 natural_t *nesting_depth,
726 vm_region_submap_info_64_t info,
727 mach_msg_type_number_t *count);
728
729extern kern_return_t vm_map_page_info(
730 vm_map_t map,
731 vm_map_offset_t offset,
732 int *disposition,
733 int *ref_count);
734
735extern kern_return_t vm_map_submap(
736 vm_map_t map,
737 vm_map_offset_t start,
738 vm_map_offset_t end,
739 vm_map_t submap,
740 vm_map_offset_t offset,
741 boolean_t use_pmap);
742
743extern void vm_map_submap_pmap_clean(
744 vm_map_t map,
745 vm_map_offset_t start,
746 vm_map_offset_t end,
747 vm_map_t sub_map,
748 vm_map_offset_t offset);
749
750
751extern vm_map_t convert_port_entry_to_map(
752 ipc_port_t port);
753
754
755extern vm_object_t convert_port_entry_to_object(
756 ipc_port_t port);
757
758
759#endif
760
761__BEGIN_DECLS
762
763
764extern vm_map_t vm_map_create(
765 pmap_t pmap,
766 vm_map_offset_t min_off,
767 vm_map_offset_t max_off,
768 boolean_t pageable);
769
770
771extern void vm_map_destroy(
772 vm_map_t map);
773
774extern void vm_map_deallocate(
775 vm_map_t map);
776
777extern vm_map_t vm_map_switch(
778 vm_map_t map);
779
780
781extern kern_return_t vm_map_protect(
782 vm_map_t map,
783 vm_map_offset_t start,
784 vm_map_offset_t end,
785 vm_prot_t new_prot,
786 boolean_t set_max);
787
788
789extern boolean_t vm_map_check_protection(
790 vm_map_t map,
791 vm_map_offset_t start,
792 vm_map_offset_t end,
793 vm_prot_t protection);
794
795
796extern kern_return_t vm_map_wire(
797 vm_map_t map,
798 vm_map_offset_t start,
799 vm_map_offset_t end,
800 vm_prot_t access_type,
801 boolean_t user_wire);
802
803
804extern kern_return_t vm_map_unwire(
805 vm_map_t map,
806 vm_map_offset_t start,
807 vm_map_offset_t end,
808 boolean_t user_wire);
809
810
811extern kern_return_t vm_map_remove(
812 vm_map_t map,
813 vm_map_offset_t start,
814 vm_map_offset_t end,
815 boolean_t flags);
816
817
818extern void vm_map_copy_discard(
819 vm_map_copy_t copy);
820
821
822extern kern_return_t vm_map_copy_overwrite(
823 vm_map_t dst_map,
824 vm_map_address_t dst_addr,
825 vm_map_copy_t copy,
826 int interruptible);
827
828
829extern kern_return_t vm_map_copyout(
830 vm_map_t dst_map,
831 vm_map_address_t *dst_addr,
832 vm_map_copy_t copy);
833
834extern kern_return_t vm_map_copyin_common(
835 vm_map_t src_map,
836 vm_map_address_t src_addr,
837 vm_map_size_t len,
838 boolean_t src_destroy,
839 boolean_t src_volatile,
840 vm_map_copy_t *copy_result,
841 boolean_t use_maxprot);
842
843
844
845
846
847
848
849
850
851
852#define vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result) \
853 vm_map_copyin_common(src_map, src_addr, len, src_destroy, \
854 FALSE, copy_result, FALSE)
855
856#define vm_map_copyin_maxprot(src_map, \
857 src_addr, len, src_destroy, copy_result) \
858 vm_map_copyin_common(src_map, src_addr, len, src_destroy, \
859 FALSE, copy_result, TRUE)
860
861
862
863
864#define vm_map_round_page(x) (((vm_map_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
865#define vm_map_trunc_page(x) ((vm_map_offset_t)(x) & ~((signed)PAGE_MASK))
866
867
868
869
870#define VM_MAP_NO_FLAGS 0x0
871#define VM_MAP_REMOVE_KUNWIRE 0x1
872#define VM_MAP_REMOVE_INTERRUPTIBLE 0x2
873#define VM_MAP_REMOVE_WAIT_FOR_KWIRE 0x4
874#define VM_MAP_REMOVE_SAVE_ENTRIES 0x8
875
876
877extern kern_return_t vm_region_clone(
878 ipc_port_t src_region,
879 ipc_port_t dst_region);
880
881extern kern_return_t vm_map_region_replace(
882 vm_map_t target_map,
883 ipc_port_t old_region,
884 ipc_port_t new_region,
885 vm_map_offset_t start,
886 vm_map_offset_t end);
887
888
889
890extern kern_return_t vm_map_get_upl(
891 vm_map_t target_map,
892 vm_map_offset_t map_offset,
893 vm_size_t *size,
894 upl_t *upl,
895 upl_page_info_array_t page_info,
896 mach_msg_type_number_t *page_infoCnt,
897 integer_t *flags,
898 integer_t force_data_sync);
899
900__END_DECLS
901
902#endif
903
904#endif
905