1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32
33#include <linux/module.h>
34#include <linux/kernel.h>
35#include <linux/sched.h>
36#include <linux/magic.h>
37#include <linux/bitops.h>
38#include <linux/errno.h>
39#include <linux/highmem.h>
40#include <linux/string.h>
41#include <linux/slab.h>
42#include <linux/pgtable.h>
43#include <asm/tlbflush.h>
44#include <linux/cpumask.h>
45#include <linux/cpu.h>
46#include <linux/vmalloc.h>
47#include <linux/preempt.h>
48#include <linux/spinlock.h>
49#include <linux/shrinker.h>
50#include <linux/types.h>
51#include <linux/debugfs.h>
52#include <linux/zsmalloc.h>
53#include <linux/zpool.h>
54#include <linux/mount.h>
55#include <linux/pseudo_fs.h>
56#include <linux/migrate.h>
57#include <linux/wait.h>
58#include <linux/pagemap.h>
59#include <linux/fs.h>
60
61#define ZSPAGE_MAGIC 0x58
62
63
64
65
66
67
68
69#define ZS_ALIGN 8
70
71
72
73
74
75#define ZS_MAX_ZSPAGE_ORDER 2
76#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
77
78#define ZS_HANDLE_SIZE (sizeof(unsigned long))
79
80
81
82
83
84
85
86
87
88
89#ifndef MAX_POSSIBLE_PHYSMEM_BITS
90#ifdef MAX_PHYSMEM_BITS
91#define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS
92#else
93
94
95
96
97#define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG
98#endif
99#endif
100
101#define _PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
102
103
104
105
106
107
108
109
110#define HANDLE_PIN_BIT 0
111
112
113
114
115
116
117
118
119#define OBJ_ALLOCATED_TAG 1
120#define OBJ_TAG_BITS 1
121#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
122#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
123
124#define FULLNESS_BITS 2
125#define CLASS_BITS 8
126#define ISOLATED_BITS 3
127#define MAGIC_VAL_BITS 8
128
129#define MAX(a, b) ((a) >= (b) ? (a) : (b))
130
131#define ZS_MIN_ALLOC_SIZE \
132 MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
133
134#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149#define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS)
150#define ZS_SIZE_CLASSES (DIV_ROUND_UP(ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE, \
151 ZS_SIZE_CLASS_DELTA) + 1)
152
153enum fullness_group {
154 ZS_EMPTY,
155 ZS_ALMOST_EMPTY,
156 ZS_ALMOST_FULL,
157 ZS_FULL,
158 NR_ZS_FULLNESS,
159};
160
161enum zs_stat_type {
162 CLASS_EMPTY,
163 CLASS_ALMOST_EMPTY,
164 CLASS_ALMOST_FULL,
165 CLASS_FULL,
166 OBJ_ALLOCATED,
167 OBJ_USED,
168 NR_ZS_STAT_TYPE,
169};
170
171struct zs_size_stat {
172 unsigned long objs[NR_ZS_STAT_TYPE];
173};
174
175#ifdef CONFIG_ZSMALLOC_STAT
176static struct dentry *zs_stat_root;
177#endif
178
179#ifdef CONFIG_COMPACTION
180static struct vfsmount *zsmalloc_mnt;
181#endif
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197static const int fullness_threshold_frac = 4;
198static size_t huge_class_size;
199
200struct size_class {
201 spinlock_t lock;
202 struct list_head fullness_list[NR_ZS_FULLNESS];
203
204
205
206
207 int size;
208 int objs_per_zspage;
209
210 int pages_per_zspage;
211
212 unsigned int index;
213 struct zs_size_stat stats;
214};
215
216
217static void SetPageHugeObject(struct page *page)
218{
219 SetPageOwnerPriv1(page);
220}
221
222static void ClearPageHugeObject(struct page *page)
223{
224 ClearPageOwnerPriv1(page);
225}
226
227static int PageHugeObject(struct page *page)
228{
229 return PageOwnerPriv1(page);
230}
231
232
233
234
235
236
237
238struct link_free {
239 union {
240
241
242
243
244 unsigned long next;
245
246
247
248 unsigned long handle;
249 };
250};
251
252struct zs_pool {
253 const char *name;
254
255 struct size_class *size_class[ZS_SIZE_CLASSES];
256 struct kmem_cache *handle_cachep;
257 struct kmem_cache *zspage_cachep;
258
259 atomic_long_t pages_allocated;
260
261 struct zs_pool_stats stats;
262
263
264 struct shrinker shrinker;
265
266#ifdef CONFIG_ZSMALLOC_STAT
267 struct dentry *stat_dentry;
268#endif
269#ifdef CONFIG_COMPACTION
270 struct inode *inode;
271 struct work_struct free_work;
272
273 struct wait_queue_head migration_wait;
274 atomic_long_t isolated_pages;
275 bool destroying;
276#endif
277};
278
279struct zspage {
280 struct {
281 unsigned int fullness:FULLNESS_BITS;
282 unsigned int class:CLASS_BITS + 1;
283 unsigned int isolated:ISOLATED_BITS;
284 unsigned int magic:MAGIC_VAL_BITS;
285 };
286 unsigned int inuse;
287 unsigned int freeobj;
288 struct page *first_page;
289 struct list_head list;
290#ifdef CONFIG_COMPACTION
291 rwlock_t lock;
292#endif
293};
294
295struct mapping_area {
296 char *vm_buf;
297 char *vm_addr;
298 enum zs_mapmode vm_mm;
299};
300
301#ifdef CONFIG_COMPACTION
302static int zs_register_migration(struct zs_pool *pool);
303static void zs_unregister_migration(struct zs_pool *pool);
304static void migrate_lock_init(struct zspage *zspage);
305static void migrate_read_lock(struct zspage *zspage);
306static void migrate_read_unlock(struct zspage *zspage);
307static void kick_deferred_free(struct zs_pool *pool);
308static void init_deferred_free(struct zs_pool *pool);
309static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
310#else
311static int zsmalloc_mount(void) { return 0; }
312static void zsmalloc_unmount(void) {}
313static int zs_register_migration(struct zs_pool *pool) { return 0; }
314static void zs_unregister_migration(struct zs_pool *pool) {}
315static void migrate_lock_init(struct zspage *zspage) {}
316static void migrate_read_lock(struct zspage *zspage) {}
317static void migrate_read_unlock(struct zspage *zspage) {}
318static void kick_deferred_free(struct zs_pool *pool) {}
319static void init_deferred_free(struct zs_pool *pool) {}
320static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}
321#endif
322
323static int create_cache(struct zs_pool *pool)
324{
325 pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
326 0, 0, NULL);
327 if (!pool->handle_cachep)
328 return 1;
329
330 pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),
331 0, 0, NULL);
332 if (!pool->zspage_cachep) {
333 kmem_cache_destroy(pool->handle_cachep);
334 pool->handle_cachep = NULL;
335 return 1;
336 }
337
338 return 0;
339}
340
341static void destroy_cache(struct zs_pool *pool)
342{
343 kmem_cache_destroy(pool->handle_cachep);
344 kmem_cache_destroy(pool->zspage_cachep);
345}
346
347static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
348{
349 return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
350 gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
351}
352
353static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
354{
355 kmem_cache_free(pool->handle_cachep, (void *)handle);
356}
357
358static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)
359{
360 return kmem_cache_zalloc(pool->zspage_cachep,
361 flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
362}
363
364static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
365{
366 kmem_cache_free(pool->zspage_cachep, zspage);
367}
368
369static void record_obj(unsigned long handle, unsigned long obj)
370{
371
372
373
374
375
376 WRITE_ONCE(*(unsigned long *)handle, obj);
377}
378
379
380
381#ifdef CONFIG_ZPOOL
382
383static void *zs_zpool_create(const char *name, gfp_t gfp,
384 const struct zpool_ops *zpool_ops,
385 struct zpool *zpool)
386{
387
388
389
390
391
392 return zs_create_pool(name);
393}
394
395static void zs_zpool_destroy(void *pool)
396{
397 zs_destroy_pool(pool);
398}
399
400static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,
401 unsigned long *handle)
402{
403 *handle = zs_malloc(pool, size, gfp);
404 return *handle ? 0 : -1;
405}
406static void zs_zpool_free(void *pool, unsigned long handle)
407{
408 zs_free(pool, handle);
409}
410
411static void *zs_zpool_map(void *pool, unsigned long handle,
412 enum zpool_mapmode mm)
413{
414 enum zs_mapmode zs_mm;
415
416 switch (mm) {
417 case ZPOOL_MM_RO:
418 zs_mm = ZS_MM_RO;
419 break;
420 case ZPOOL_MM_WO:
421 zs_mm = ZS_MM_WO;
422 break;
423 case ZPOOL_MM_RW:
424 default:
425 zs_mm = ZS_MM_RW;
426 break;
427 }
428
429 return zs_map_object(pool, handle, zs_mm);
430}
431static void zs_zpool_unmap(void *pool, unsigned long handle)
432{
433 zs_unmap_object(pool, handle);
434}
435
436static u64 zs_zpool_total_size(void *pool)
437{
438 return zs_get_total_pages(pool) << PAGE_SHIFT;
439}
440
441static struct zpool_driver zs_zpool_driver = {
442 .type = "zsmalloc",
443 .owner = THIS_MODULE,
444 .create = zs_zpool_create,
445 .destroy = zs_zpool_destroy,
446 .malloc_support_movable = true,
447 .malloc = zs_zpool_malloc,
448 .free = zs_zpool_free,
449 .map = zs_zpool_map,
450 .unmap = zs_zpool_unmap,
451 .total_size = zs_zpool_total_size,
452};
453
454MODULE_ALIAS("zpool-zsmalloc");
455#endif
456
457
458static DEFINE_PER_CPU(struct mapping_area, zs_map_area);
459
460static bool is_zspage_isolated(struct zspage *zspage)
461{
462 return zspage->isolated;
463}
464
465static __maybe_unused int is_first_page(struct page *page)
466{
467 return PagePrivate(page);
468}
469
470
471static inline int get_zspage_inuse(struct zspage *zspage)
472{
473 return zspage->inuse;
474}
475
476
477static inline void mod_zspage_inuse(struct zspage *zspage, int val)
478{
479 zspage->inuse += val;
480}
481
482static inline struct page *get_first_page(struct zspage *zspage)
483{
484 struct page *first_page = zspage->first_page;
485
486 VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
487 return first_page;
488}
489
490static inline int get_first_obj_offset(struct page *page)
491{
492 return page->units;
493}
494
495static inline void set_first_obj_offset(struct page *page, int offset)
496{
497 page->units = offset;
498}
499
500static inline unsigned int get_freeobj(struct zspage *zspage)
501{
502 return zspage->freeobj;
503}
504
505static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
506{
507 zspage->freeobj = obj;
508}
509
510static void get_zspage_mapping(struct zspage *zspage,
511 unsigned int *class_idx,
512 enum fullness_group *fullness)
513{
514 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
515
516 *fullness = zspage->fullness;
517 *class_idx = zspage->class;
518}
519
520static void set_zspage_mapping(struct zspage *zspage,
521 unsigned int class_idx,
522 enum fullness_group fullness)
523{
524 zspage->class = class_idx;
525 zspage->fullness = fullness;
526}
527
528
529
530
531
532
533
534
535static int get_size_class_index(int size)
536{
537 int idx = 0;
538
539 if (likely(size > ZS_MIN_ALLOC_SIZE))
540 idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE,
541 ZS_SIZE_CLASS_DELTA);
542
543 return min_t(int, ZS_SIZE_CLASSES - 1, idx);
544}
545
546
547static inline void zs_stat_inc(struct size_class *class,
548 int type, unsigned long cnt)
549{
550 class->stats.objs[type] += cnt;
551}
552
553
554static inline void zs_stat_dec(struct size_class *class,
555 int type, unsigned long cnt)
556{
557 class->stats.objs[type] -= cnt;
558}
559
560
561static inline unsigned long zs_stat_get(struct size_class *class,
562 int type)
563{
564 return class->stats.objs[type];
565}
566
567#ifdef CONFIG_ZSMALLOC_STAT
568
569static void __init zs_stat_init(void)
570{
571 if (!debugfs_initialized()) {
572 pr_warn("debugfs not available, stat dir not created\n");
573 return;
574 }
575
576 zs_stat_root = debugfs_create_dir("zsmalloc", NULL);
577}
578
579static void __exit zs_stat_exit(void)
580{
581 debugfs_remove_recursive(zs_stat_root);
582}
583
584static unsigned long zs_can_compact(struct size_class *class);
585
586static int zs_stats_size_show(struct seq_file *s, void *v)
587{
588 int i;
589 struct zs_pool *pool = s->private;
590 struct size_class *class;
591 int objs_per_zspage;
592 unsigned long class_almost_full, class_almost_empty;
593 unsigned long obj_allocated, obj_used, pages_used, freeable;
594 unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
595 unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
596 unsigned long total_freeable = 0;
597
598 seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
599 "class", "size", "almost_full", "almost_empty",
600 "obj_allocated", "obj_used", "pages_used",
601 "pages_per_zspage", "freeable");
602
603 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
604 class = pool->size_class[i];
605
606 if (class->index != i)
607 continue;
608
609 spin_lock(&class->lock);
610 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
611 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
612 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
613 obj_used = zs_stat_get(class, OBJ_USED);
614 freeable = zs_can_compact(class);
615 spin_unlock(&class->lock);
616
617 objs_per_zspage = class->objs_per_zspage;
618 pages_used = obj_allocated / objs_per_zspage *
619 class->pages_per_zspage;
620
621 seq_printf(s, " %5u %5u %11lu %12lu %13lu"
622 " %10lu %10lu %16d %8lu\n",
623 i, class->size, class_almost_full, class_almost_empty,
624 obj_allocated, obj_used, pages_used,
625 class->pages_per_zspage, freeable);
626
627 total_class_almost_full += class_almost_full;
628 total_class_almost_empty += class_almost_empty;
629 total_objs += obj_allocated;
630 total_used_objs += obj_used;
631 total_pages += pages_used;
632 total_freeable += freeable;
633 }
634
635 seq_puts(s, "\n");
636 seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
637 "Total", "", total_class_almost_full,
638 total_class_almost_empty, total_objs,
639 total_used_objs, total_pages, "", total_freeable);
640
641 return 0;
642}
643DEFINE_SHOW_ATTRIBUTE(zs_stats_size);
644
645static void zs_pool_stat_create(struct zs_pool *pool, const char *name)
646{
647 if (!zs_stat_root) {
648 pr_warn("no root stat dir, not creating <%s> stat dir\n", name);
649 return;
650 }
651
652 pool->stat_dentry = debugfs_create_dir(name, zs_stat_root);
653
654 debugfs_create_file("classes", S_IFREG | 0444, pool->stat_dentry, pool,
655 &zs_stats_size_fops);
656}
657
658static void zs_pool_stat_destroy(struct zs_pool *pool)
659{
660 debugfs_remove_recursive(pool->stat_dentry);
661}
662
663#else
664static void __init zs_stat_init(void)
665{
666}
667
668static void __exit zs_stat_exit(void)
669{
670}
671
672static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name)
673{
674}
675
676static inline void zs_pool_stat_destroy(struct zs_pool *pool)
677{
678}
679#endif
680
681
682
683
684
685
686
687
688
689static enum fullness_group get_fullness_group(struct size_class *class,
690 struct zspage *zspage)
691{
692 int inuse, objs_per_zspage;
693 enum fullness_group fg;
694
695 inuse = get_zspage_inuse(zspage);
696 objs_per_zspage = class->objs_per_zspage;
697
698 if (inuse == 0)
699 fg = ZS_EMPTY;
700 else if (inuse == objs_per_zspage)
701 fg = ZS_FULL;
702 else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac)
703 fg = ZS_ALMOST_EMPTY;
704 else
705 fg = ZS_ALMOST_FULL;
706
707 return fg;
708}
709
710
711
712
713
714
715
716static void insert_zspage(struct size_class *class,
717 struct zspage *zspage,
718 enum fullness_group fullness)
719{
720 struct zspage *head;
721
722 zs_stat_inc(class, fullness, 1);
723 head = list_first_entry_or_null(&class->fullness_list[fullness],
724 struct zspage, list);
725
726
727
728
729 if (head && get_zspage_inuse(zspage) < get_zspage_inuse(head))
730 list_add(&zspage->list, &head->list);
731 else
732 list_add(&zspage->list, &class->fullness_list[fullness]);
733}
734
735
736
737
738
739static void remove_zspage(struct size_class *class,
740 struct zspage *zspage,
741 enum fullness_group fullness)
742{
743 VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
744 VM_BUG_ON(is_zspage_isolated(zspage));
745
746 list_del_init(&zspage->list);
747 zs_stat_dec(class, fullness, 1);
748}
749
750
751
752
753
754
755
756
757
758
759static enum fullness_group fix_fullness_group(struct size_class *class,
760 struct zspage *zspage)
761{
762 int class_idx;
763 enum fullness_group currfg, newfg;
764
765 get_zspage_mapping(zspage, &class_idx, &currfg);
766 newfg = get_fullness_group(class, zspage);
767 if (newfg == currfg)
768 goto out;
769
770 if (!is_zspage_isolated(zspage)) {
771 remove_zspage(class, zspage, currfg);
772 insert_zspage(class, zspage, newfg);
773 }
774
775 set_zspage_mapping(zspage, class_idx, newfg);
776
777out:
778 return newfg;
779}
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794static int get_pages_per_zspage(int class_size)
795{
796 int i, max_usedpc = 0;
797
798 int max_usedpc_order = 1;
799
800 for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) {
801 int zspage_size;
802 int waste, usedpc;
803
804 zspage_size = i * PAGE_SIZE;
805 waste = zspage_size % class_size;
806 usedpc = (zspage_size - waste) * 100 / zspage_size;
807
808 if (usedpc > max_usedpc) {
809 max_usedpc = usedpc;
810 max_usedpc_order = i;
811 }
812 }
813
814 return max_usedpc_order;
815}
816
817static struct zspage *get_zspage(struct page *page)
818{
819 struct zspage *zspage = (struct zspage *)page_private(page);
820
821 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
822 return zspage;
823}
824
825static struct page *get_next_page(struct page *page)
826{
827 if (unlikely(PageHugeObject(page)))
828 return NULL;
829
830 return page->freelist;
831}
832
833
834
835
836
837
838
839static void obj_to_location(unsigned long obj, struct page **page,
840 unsigned int *obj_idx)
841{
842 obj >>= OBJ_TAG_BITS;
843 *page = pfn_to_page(obj >> OBJ_INDEX_BITS);
844 *obj_idx = (obj & OBJ_INDEX_MASK);
845}
846
847
848
849
850
851
852static unsigned long location_to_obj(struct page *page, unsigned int obj_idx)
853{
854 unsigned long obj;
855
856 obj = page_to_pfn(page) << OBJ_INDEX_BITS;
857 obj |= obj_idx & OBJ_INDEX_MASK;
858 obj <<= OBJ_TAG_BITS;
859
860 return obj;
861}
862
863static unsigned long handle_to_obj(unsigned long handle)
864{
865 return *(unsigned long *)handle;
866}
867
868static unsigned long obj_to_head(struct page *page, void *obj)
869{
870 if (unlikely(PageHugeObject(page))) {
871 VM_BUG_ON_PAGE(!is_first_page(page), page);
872 return page->index;
873 } else
874 return *(unsigned long *)obj;
875}
876
877static inline int testpin_tag(unsigned long handle)
878{
879 return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
880}
881
882static inline int trypin_tag(unsigned long handle)
883{
884 return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
885}
886
887static void pin_tag(unsigned long handle) __acquires(bitlock)
888{
889 bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
890}
891
892static void unpin_tag(unsigned long handle) __releases(bitlock)
893{
894 bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
895}
896
897static void reset_page(struct page *page)
898{
899 __ClearPageMovable(page);
900 ClearPagePrivate(page);
901 set_page_private(page, 0);
902 page_mapcount_reset(page);
903 ClearPageHugeObject(page);
904 page->freelist = NULL;
905}
906
907static int trylock_zspage(struct zspage *zspage)
908{
909 struct page *cursor, *fail;
910
911 for (cursor = get_first_page(zspage); cursor != NULL; cursor =
912 get_next_page(cursor)) {
913 if (!trylock_page(cursor)) {
914 fail = cursor;
915 goto unlock;
916 }
917 }
918
919 return 1;
920unlock:
921 for (cursor = get_first_page(zspage); cursor != fail; cursor =
922 get_next_page(cursor))
923 unlock_page(cursor);
924
925 return 0;
926}
927
928static void __free_zspage(struct zs_pool *pool, struct size_class *class,
929 struct zspage *zspage)
930{
931 struct page *page, *next;
932 enum fullness_group fg;
933 unsigned int class_idx;
934
935 get_zspage_mapping(zspage, &class_idx, &fg);
936
937 assert_spin_locked(&class->lock);
938
939 VM_BUG_ON(get_zspage_inuse(zspage));
940 VM_BUG_ON(fg != ZS_EMPTY);
941
942 next = page = get_first_page(zspage);
943 do {
944 VM_BUG_ON_PAGE(!PageLocked(page), page);
945 next = get_next_page(page);
946 reset_page(page);
947 unlock_page(page);
948 dec_zone_page_state(page, NR_ZSPAGES);
949 put_page(page);
950 page = next;
951 } while (page != NULL);
952
953 cache_free_zspage(pool, zspage);
954
955 zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
956 atomic_long_sub(class->pages_per_zspage,
957 &pool->pages_allocated);
958}
959
960static void free_zspage(struct zs_pool *pool, struct size_class *class,
961 struct zspage *zspage)
962{
963 VM_BUG_ON(get_zspage_inuse(zspage));
964 VM_BUG_ON(list_empty(&zspage->list));
965
966 if (!trylock_zspage(zspage)) {
967 kick_deferred_free(pool);
968 return;
969 }
970
971 remove_zspage(class, zspage, ZS_EMPTY);
972 __free_zspage(pool, class, zspage);
973}
974
975
976static void init_zspage(struct size_class *class, struct zspage *zspage)
977{
978 unsigned int freeobj = 1;
979 unsigned long off = 0;
980 struct page *page = get_first_page(zspage);
981
982 while (page) {
983 struct page *next_page;
984 struct link_free *link;
985 void *vaddr;
986
987 set_first_obj_offset(page, off);
988
989 vaddr = kmap_atomic(page);
990 link = (struct link_free *)vaddr + off / sizeof(*link);
991
992 while ((off += class->size) < PAGE_SIZE) {
993 link->next = freeobj++ << OBJ_TAG_BITS;
994 link += class->size / sizeof(*link);
995 }
996
997
998
999
1000
1001
1002 next_page = get_next_page(page);
1003 if (next_page) {
1004 link->next = freeobj++ << OBJ_TAG_BITS;
1005 } else {
1006
1007
1008
1009
1010 link->next = -1UL << OBJ_TAG_BITS;
1011 }
1012 kunmap_atomic(vaddr);
1013 page = next_page;
1014 off %= PAGE_SIZE;
1015 }
1016
1017 set_freeobj(zspage, 0);
1018}
1019
1020static void create_page_chain(struct size_class *class, struct zspage *zspage,
1021 struct page *pages[])
1022{
1023 int i;
1024 struct page *page;
1025 struct page *prev_page = NULL;
1026 int nr_pages = class->pages_per_zspage;
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036 for (i = 0; i < nr_pages; i++) {
1037 page = pages[i];
1038 set_page_private(page, (unsigned long)zspage);
1039 page->freelist = NULL;
1040 if (i == 0) {
1041 zspage->first_page = page;
1042 SetPagePrivate(page);
1043 if (unlikely(class->objs_per_zspage == 1 &&
1044 class->pages_per_zspage == 1))
1045 SetPageHugeObject(page);
1046 } else {
1047 prev_page->freelist = page;
1048 }
1049 prev_page = page;
1050 }
1051}
1052
1053
1054
1055
1056static struct zspage *alloc_zspage(struct zs_pool *pool,
1057 struct size_class *class,
1058 gfp_t gfp)
1059{
1060 int i;
1061 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE];
1062 struct zspage *zspage = cache_alloc_zspage(pool, gfp);
1063
1064 if (!zspage)
1065 return NULL;
1066
1067 zspage->magic = ZSPAGE_MAGIC;
1068 migrate_lock_init(zspage);
1069
1070 for (i = 0; i < class->pages_per_zspage; i++) {
1071 struct page *page;
1072
1073 page = alloc_page(gfp);
1074 if (!page) {
1075 while (--i >= 0) {
1076 dec_zone_page_state(pages[i], NR_ZSPAGES);
1077 __free_page(pages[i]);
1078 }
1079 cache_free_zspage(pool, zspage);
1080 return NULL;
1081 }
1082
1083 inc_zone_page_state(page, NR_ZSPAGES);
1084 pages[i] = page;
1085 }
1086
1087 create_page_chain(class, zspage, pages);
1088 init_zspage(class, zspage);
1089
1090 return zspage;
1091}
1092
1093static struct zspage *find_get_zspage(struct size_class *class)
1094{
1095 int i;
1096 struct zspage *zspage;
1097
1098 for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) {
1099 zspage = list_first_entry_or_null(&class->fullness_list[i],
1100 struct zspage, list);
1101 if (zspage)
1102 break;
1103 }
1104
1105 return zspage;
1106}
1107
1108static inline int __zs_cpu_up(struct mapping_area *area)
1109{
1110
1111
1112
1113
1114 if (area->vm_buf)
1115 return 0;
1116 area->vm_buf = kmalloc(ZS_MAX_ALLOC_SIZE, GFP_KERNEL);
1117 if (!area->vm_buf)
1118 return -ENOMEM;
1119 return 0;
1120}
1121
1122static inline void __zs_cpu_down(struct mapping_area *area)
1123{
1124 kfree(area->vm_buf);
1125 area->vm_buf = NULL;
1126}
1127
1128static void *__zs_map_object(struct mapping_area *area,
1129 struct page *pages[2], int off, int size)
1130{
1131 int sizes[2];
1132 void *addr;
1133 char *buf = area->vm_buf;
1134
1135
1136 pagefault_disable();
1137
1138
1139 if (area->vm_mm == ZS_MM_WO)
1140 goto out;
1141
1142 sizes[0] = PAGE_SIZE - off;
1143 sizes[1] = size - sizes[0];
1144
1145
1146 addr = kmap_atomic(pages[0]);
1147 memcpy(buf, addr + off, sizes[0]);
1148 kunmap_atomic(addr);
1149 addr = kmap_atomic(pages[1]);
1150 memcpy(buf + sizes[0], addr, sizes[1]);
1151 kunmap_atomic(addr);
1152out:
1153 return area->vm_buf;
1154}
1155
1156static void __zs_unmap_object(struct mapping_area *area,
1157 struct page *pages[2], int off, int size)
1158{
1159 int sizes[2];
1160 void *addr;
1161 char *buf;
1162
1163
1164 if (area->vm_mm == ZS_MM_RO)
1165 goto out;
1166
1167 buf = area->vm_buf;
1168 buf = buf + ZS_HANDLE_SIZE;
1169 size -= ZS_HANDLE_SIZE;
1170 off += ZS_HANDLE_SIZE;
1171
1172 sizes[0] = PAGE_SIZE - off;
1173 sizes[1] = size - sizes[0];
1174
1175
1176 addr = kmap_atomic(pages[0]);
1177 memcpy(addr + off, buf, sizes[0]);
1178 kunmap_atomic(addr);
1179 addr = kmap_atomic(pages[1]);
1180 memcpy(addr, buf + sizes[0], sizes[1]);
1181 kunmap_atomic(addr);
1182
1183out:
1184
1185 pagefault_enable();
1186}
1187
1188static int zs_cpu_prepare(unsigned int cpu)
1189{
1190 struct mapping_area *area;
1191
1192 area = &per_cpu(zs_map_area, cpu);
1193 return __zs_cpu_up(area);
1194}
1195
1196static int zs_cpu_dead(unsigned int cpu)
1197{
1198 struct mapping_area *area;
1199
1200 area = &per_cpu(zs_map_area, cpu);
1201 __zs_cpu_down(area);
1202 return 0;
1203}
1204
1205static bool can_merge(struct size_class *prev, int pages_per_zspage,
1206 int objs_per_zspage)
1207{
1208 if (prev->pages_per_zspage == pages_per_zspage &&
1209 prev->objs_per_zspage == objs_per_zspage)
1210 return true;
1211
1212 return false;
1213}
1214
1215static bool zspage_full(struct size_class *class, struct zspage *zspage)
1216{
1217 return get_zspage_inuse(zspage) == class->objs_per_zspage;
1218}
1219
1220unsigned long zs_get_total_pages(struct zs_pool *pool)
1221{
1222 return atomic_long_read(&pool->pages_allocated);
1223}
1224EXPORT_SYMBOL_GPL(zs_get_total_pages);
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241void *zs_map_object(struct zs_pool *pool, unsigned long handle,
1242 enum zs_mapmode mm)
1243{
1244 struct zspage *zspage;
1245 struct page *page;
1246 unsigned long obj, off;
1247 unsigned int obj_idx;
1248
1249 unsigned int class_idx;
1250 enum fullness_group fg;
1251 struct size_class *class;
1252 struct mapping_area *area;
1253 struct page *pages[2];
1254 void *ret;
1255
1256
1257
1258
1259
1260
1261 BUG_ON(in_interrupt());
1262
1263
1264 pin_tag(handle);
1265
1266 obj = handle_to_obj(handle);
1267 obj_to_location(obj, &page, &obj_idx);
1268 zspage = get_zspage(page);
1269
1270
1271 migrate_read_lock(zspage);
1272
1273 get_zspage_mapping(zspage, &class_idx, &fg);
1274 class = pool->size_class[class_idx];
1275 off = (class->size * obj_idx) & ~PAGE_MASK;
1276
1277 area = &get_cpu_var(zs_map_area);
1278 area->vm_mm = mm;
1279 if (off + class->size <= PAGE_SIZE) {
1280
1281 area->vm_addr = kmap_atomic(page);
1282 ret = area->vm_addr + off;
1283 goto out;
1284 }
1285
1286
1287 pages[0] = page;
1288 pages[1] = get_next_page(page);
1289 BUG_ON(!pages[1]);
1290
1291 ret = __zs_map_object(area, pages, off, class->size);
1292out:
1293 if (likely(!PageHugeObject(page)))
1294 ret += ZS_HANDLE_SIZE;
1295
1296 return ret;
1297}
1298EXPORT_SYMBOL_GPL(zs_map_object);
1299
1300void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
1301{
1302 struct zspage *zspage;
1303 struct page *page;
1304 unsigned long obj, off;
1305 unsigned int obj_idx;
1306
1307 unsigned int class_idx;
1308 enum fullness_group fg;
1309 struct size_class *class;
1310 struct mapping_area *area;
1311
1312 obj = handle_to_obj(handle);
1313 obj_to_location(obj, &page, &obj_idx);
1314 zspage = get_zspage(page);
1315 get_zspage_mapping(zspage, &class_idx, &fg);
1316 class = pool->size_class[class_idx];
1317 off = (class->size * obj_idx) & ~PAGE_MASK;
1318
1319 area = this_cpu_ptr(&zs_map_area);
1320 if (off + class->size <= PAGE_SIZE)
1321 kunmap_atomic(area->vm_addr);
1322 else {
1323 struct page *pages[2];
1324
1325 pages[0] = page;
1326 pages[1] = get_next_page(page);
1327 BUG_ON(!pages[1]);
1328
1329 __zs_unmap_object(area, pages, off, class->size);
1330 }
1331 put_cpu_var(zs_map_area);
1332
1333 migrate_read_unlock(zspage);
1334 unpin_tag(handle);
1335}
1336EXPORT_SYMBOL_GPL(zs_unmap_object);
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351size_t zs_huge_class_size(struct zs_pool *pool)
1352{
1353 return huge_class_size;
1354}
1355EXPORT_SYMBOL_GPL(zs_huge_class_size);
1356
1357static unsigned long obj_malloc(struct size_class *class,
1358 struct zspage *zspage, unsigned long handle)
1359{
1360 int i, nr_page, offset;
1361 unsigned long obj;
1362 struct link_free *link;
1363
1364 struct page *m_page;
1365 unsigned long m_offset;
1366 void *vaddr;
1367
1368 handle |= OBJ_ALLOCATED_TAG;
1369 obj = get_freeobj(zspage);
1370
1371 offset = obj * class->size;
1372 nr_page = offset >> PAGE_SHIFT;
1373 m_offset = offset & ~PAGE_MASK;
1374 m_page = get_first_page(zspage);
1375
1376 for (i = 0; i < nr_page; i++)
1377 m_page = get_next_page(m_page);
1378
1379 vaddr = kmap_atomic(m_page);
1380 link = (struct link_free *)vaddr + m_offset / sizeof(*link);
1381 set_freeobj(zspage, link->next >> OBJ_TAG_BITS);
1382 if (likely(!PageHugeObject(m_page)))
1383
1384 link->handle = handle;
1385 else
1386
1387 zspage->first_page->index = handle;
1388
1389 kunmap_atomic(vaddr);
1390 mod_zspage_inuse(zspage, 1);
1391 zs_stat_inc(class, OBJ_USED, 1);
1392
1393 obj = location_to_obj(m_page, obj);
1394
1395 return obj;
1396}
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
1410{
1411 unsigned long handle, obj;
1412 struct size_class *class;
1413 enum fullness_group newfg;
1414 struct zspage *zspage;
1415
1416 if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE))
1417 return 0;
1418
1419 handle = cache_alloc_handle(pool, gfp);
1420 if (!handle)
1421 return 0;
1422
1423
1424 size += ZS_HANDLE_SIZE;
1425 class = pool->size_class[get_size_class_index(size)];
1426
1427 spin_lock(&class->lock);
1428 zspage = find_get_zspage(class);
1429 if (likely(zspage)) {
1430 obj = obj_malloc(class, zspage, handle);
1431
1432 fix_fullness_group(class, zspage);
1433 record_obj(handle, obj);
1434 spin_unlock(&class->lock);
1435
1436 return handle;
1437 }
1438
1439 spin_unlock(&class->lock);
1440
1441 zspage = alloc_zspage(pool, class, gfp);
1442 if (!zspage) {
1443 cache_free_handle(pool, handle);
1444 return 0;
1445 }
1446
1447 spin_lock(&class->lock);
1448 obj = obj_malloc(class, zspage, handle);
1449 newfg = get_fullness_group(class, zspage);
1450 insert_zspage(class, zspage, newfg);
1451 set_zspage_mapping(zspage, class->index, newfg);
1452 record_obj(handle, obj);
1453 atomic_long_add(class->pages_per_zspage,
1454 &pool->pages_allocated);
1455 zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage);
1456
1457
1458 SetZsPageMovable(pool, zspage);
1459 spin_unlock(&class->lock);
1460
1461 return handle;
1462}
1463EXPORT_SYMBOL_GPL(zs_malloc);
1464
1465static void obj_free(struct size_class *class, unsigned long obj)
1466{
1467 struct link_free *link;
1468 struct zspage *zspage;
1469 struct page *f_page;
1470 unsigned long f_offset;
1471 unsigned int f_objidx;
1472 void *vaddr;
1473
1474 obj_to_location(obj, &f_page, &f_objidx);
1475 f_offset = (class->size * f_objidx) & ~PAGE_MASK;
1476 zspage = get_zspage(f_page);
1477
1478 vaddr = kmap_atomic(f_page);
1479
1480
1481 link = (struct link_free *)(vaddr + f_offset);
1482 link->next = get_freeobj(zspage) << OBJ_TAG_BITS;
1483 kunmap_atomic(vaddr);
1484 set_freeobj(zspage, f_objidx);
1485 mod_zspage_inuse(zspage, -1);
1486 zs_stat_dec(class, OBJ_USED, 1);
1487}
1488
1489void zs_free(struct zs_pool *pool, unsigned long handle)
1490{
1491 struct zspage *zspage;
1492 struct page *f_page;
1493 unsigned long obj;
1494 unsigned int f_objidx;
1495 int class_idx;
1496 struct size_class *class;
1497 enum fullness_group fullness;
1498 bool isolated;
1499
1500 if (unlikely(!handle))
1501 return;
1502
1503 pin_tag(handle);
1504 obj = handle_to_obj(handle);
1505 obj_to_location(obj, &f_page, &f_objidx);
1506 zspage = get_zspage(f_page);
1507
1508 migrate_read_lock(zspage);
1509
1510 get_zspage_mapping(zspage, &class_idx, &fullness);
1511 class = pool->size_class[class_idx];
1512
1513 spin_lock(&class->lock);
1514 obj_free(class, obj);
1515 fullness = fix_fullness_group(class, zspage);
1516 if (fullness != ZS_EMPTY) {
1517 migrate_read_unlock(zspage);
1518 goto out;
1519 }
1520
1521 isolated = is_zspage_isolated(zspage);
1522 migrate_read_unlock(zspage);
1523
1524 if (likely(!isolated))
1525 free_zspage(pool, class, zspage);
1526out:
1527
1528 spin_unlock(&class->lock);
1529 unpin_tag(handle);
1530 cache_free_handle(pool, handle);
1531}
1532EXPORT_SYMBOL_GPL(zs_free);
1533
1534static void zs_object_copy(struct size_class *class, unsigned long dst,
1535 unsigned long src)
1536{
1537 struct page *s_page, *d_page;
1538 unsigned int s_objidx, d_objidx;
1539 unsigned long s_off, d_off;
1540 void *s_addr, *d_addr;
1541 int s_size, d_size, size;
1542 int written = 0;
1543
1544 s_size = d_size = class->size;
1545
1546 obj_to_location(src, &s_page, &s_objidx);
1547 obj_to_location(dst, &d_page, &d_objidx);
1548
1549 s_off = (class->size * s_objidx) & ~PAGE_MASK;
1550 d_off = (class->size * d_objidx) & ~PAGE_MASK;
1551
1552 if (s_off + class->size > PAGE_SIZE)
1553 s_size = PAGE_SIZE - s_off;
1554
1555 if (d_off + class->size > PAGE_SIZE)
1556 d_size = PAGE_SIZE - d_off;
1557
1558 s_addr = kmap_atomic(s_page);
1559 d_addr = kmap_atomic(d_page);
1560
1561 while (1) {
1562 size = min(s_size, d_size);
1563 memcpy(d_addr + d_off, s_addr + s_off, size);
1564 written += size;
1565
1566 if (written == class->size)
1567 break;
1568
1569 s_off += size;
1570 s_size -= size;
1571 d_off += size;
1572 d_size -= size;
1573
1574 if (s_off >= PAGE_SIZE) {
1575 kunmap_atomic(d_addr);
1576 kunmap_atomic(s_addr);
1577 s_page = get_next_page(s_page);
1578 s_addr = kmap_atomic(s_page);
1579 d_addr = kmap_atomic(d_page);
1580 s_size = class->size - written;
1581 s_off = 0;
1582 }
1583
1584 if (d_off >= PAGE_SIZE) {
1585 kunmap_atomic(d_addr);
1586 d_page = get_next_page(d_page);
1587 d_addr = kmap_atomic(d_page);
1588 d_size = class->size - written;
1589 d_off = 0;
1590 }
1591 }
1592
1593 kunmap_atomic(d_addr);
1594 kunmap_atomic(s_addr);
1595}
1596
1597
1598
1599
1600
1601static unsigned long find_alloced_obj(struct size_class *class,
1602 struct page *page, int *obj_idx)
1603{
1604 unsigned long head;
1605 int offset = 0;
1606 int index = *obj_idx;
1607 unsigned long handle = 0;
1608 void *addr = kmap_atomic(page);
1609
1610 offset = get_first_obj_offset(page);
1611 offset += class->size * index;
1612
1613 while (offset < PAGE_SIZE) {
1614 head = obj_to_head(page, addr + offset);
1615 if (head & OBJ_ALLOCATED_TAG) {
1616 handle = head & ~OBJ_ALLOCATED_TAG;
1617 if (trypin_tag(handle))
1618 break;
1619 handle = 0;
1620 }
1621
1622 offset += class->size;
1623 index++;
1624 }
1625
1626 kunmap_atomic(addr);
1627
1628 *obj_idx = index;
1629
1630 return handle;
1631}
1632
1633struct zs_compact_control {
1634
1635 struct page *s_page;
1636
1637
1638 struct page *d_page;
1639
1640
1641 int obj_idx;
1642};
1643
1644static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
1645 struct zs_compact_control *cc)
1646{
1647 unsigned long used_obj, free_obj;
1648 unsigned long handle;
1649 struct page *s_page = cc->s_page;
1650 struct page *d_page = cc->d_page;
1651 int obj_idx = cc->obj_idx;
1652 int ret = 0;
1653
1654 while (1) {
1655 handle = find_alloced_obj(class, s_page, &obj_idx);
1656 if (!handle) {
1657 s_page = get_next_page(s_page);
1658 if (!s_page)
1659 break;
1660 obj_idx = 0;
1661 continue;
1662 }
1663
1664
1665 if (zspage_full(class, get_zspage(d_page))) {
1666 unpin_tag(handle);
1667 ret = -ENOMEM;
1668 break;
1669 }
1670
1671 used_obj = handle_to_obj(handle);
1672 free_obj = obj_malloc(class, get_zspage(d_page), handle);
1673 zs_object_copy(class, free_obj, used_obj);
1674 obj_idx++;
1675
1676
1677
1678
1679
1680
1681 free_obj |= BIT(HANDLE_PIN_BIT);
1682 record_obj(handle, free_obj);
1683 unpin_tag(handle);
1684 obj_free(class, used_obj);
1685 }
1686
1687
1688 cc->s_page = s_page;
1689 cc->obj_idx = obj_idx;
1690
1691 return ret;
1692}
1693
1694static struct zspage *isolate_zspage(struct size_class *class, bool source)
1695{
1696 int i;
1697 struct zspage *zspage;
1698 enum fullness_group fg[2] = {ZS_ALMOST_EMPTY, ZS_ALMOST_FULL};
1699
1700 if (!source) {
1701 fg[0] = ZS_ALMOST_FULL;
1702 fg[1] = ZS_ALMOST_EMPTY;
1703 }
1704
1705 for (i = 0; i < 2; i++) {
1706 zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],
1707 struct zspage, list);
1708 if (zspage) {
1709 VM_BUG_ON(is_zspage_isolated(zspage));
1710 remove_zspage(class, zspage, fg[i]);
1711 return zspage;
1712 }
1713 }
1714
1715 return zspage;
1716}
1717
1718
1719
1720
1721
1722
1723
1724
1725static enum fullness_group putback_zspage(struct size_class *class,
1726 struct zspage *zspage)
1727{
1728 enum fullness_group fullness;
1729
1730 VM_BUG_ON(is_zspage_isolated(zspage));
1731
1732 fullness = get_fullness_group(class, zspage);
1733 insert_zspage(class, zspage, fullness);
1734 set_zspage_mapping(zspage, class->index, fullness);
1735
1736 return fullness;
1737}
1738
1739#ifdef CONFIG_COMPACTION
1740
1741
1742
1743
1744static void lock_zspage(struct zspage *zspage)
1745{
1746 struct page *page = get_first_page(zspage);
1747
1748 do {
1749 lock_page(page);
1750 } while ((page = get_next_page(page)) != NULL);
1751}
1752
1753static int zs_init_fs_context(struct fs_context *fc)
1754{
1755 return init_pseudo(fc, ZSMALLOC_MAGIC) ? 0 : -ENOMEM;
1756}
1757
1758static struct file_system_type zsmalloc_fs = {
1759 .name = "zsmalloc",
1760 .init_fs_context = zs_init_fs_context,
1761 .kill_sb = kill_anon_super,
1762};
1763
1764static int zsmalloc_mount(void)
1765{
1766 int ret = 0;
1767
1768 zsmalloc_mnt = kern_mount(&zsmalloc_fs);
1769 if (IS_ERR(zsmalloc_mnt))
1770 ret = PTR_ERR(zsmalloc_mnt);
1771
1772 return ret;
1773}
1774
1775static void zsmalloc_unmount(void)
1776{
1777 kern_unmount(zsmalloc_mnt);
1778}
1779
1780static void migrate_lock_init(struct zspage *zspage)
1781{
1782 rwlock_init(&zspage->lock);
1783}
1784
1785static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock)
1786{
1787 read_lock(&zspage->lock);
1788}
1789
1790static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock)
1791{
1792 read_unlock(&zspage->lock);
1793}
1794
1795static void migrate_write_lock(struct zspage *zspage)
1796{
1797 write_lock(&zspage->lock);
1798}
1799
1800static void migrate_write_unlock(struct zspage *zspage)
1801{
1802 write_unlock(&zspage->lock);
1803}
1804
1805
1806static void inc_zspage_isolation(struct zspage *zspage)
1807{
1808 zspage->isolated++;
1809}
1810
1811static void dec_zspage_isolation(struct zspage *zspage)
1812{
1813 zspage->isolated--;
1814}
1815
1816static void putback_zspage_deferred(struct zs_pool *pool,
1817 struct size_class *class,
1818 struct zspage *zspage)
1819{
1820 enum fullness_group fg;
1821
1822 fg = putback_zspage(class, zspage);
1823 if (fg == ZS_EMPTY)
1824 schedule_work(&pool->free_work);
1825
1826}
1827
1828static inline void zs_pool_dec_isolated(struct zs_pool *pool)
1829{
1830 VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
1831 atomic_long_dec(&pool->isolated_pages);
1832
1833
1834
1835
1836
1837 if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
1838 wake_up_all(&pool->migration_wait);
1839}
1840
1841static void replace_sub_page(struct size_class *class, struct zspage *zspage,
1842 struct page *newpage, struct page *oldpage)
1843{
1844 struct page *page;
1845 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, };
1846 int idx = 0;
1847
1848 page = get_first_page(zspage);
1849 do {
1850 if (page == oldpage)
1851 pages[idx] = newpage;
1852 else
1853 pages[idx] = page;
1854 idx++;
1855 } while ((page = get_next_page(page)) != NULL);
1856
1857 create_page_chain(class, zspage, pages);
1858 set_first_obj_offset(newpage, get_first_obj_offset(oldpage));
1859 if (unlikely(PageHugeObject(oldpage)))
1860 newpage->index = oldpage->index;
1861 __SetPageMovable(newpage, page_mapping(oldpage));
1862}
1863
1864static bool zs_page_isolate(struct page *page, isolate_mode_t mode)
1865{
1866 struct zs_pool *pool;
1867 struct size_class *class;
1868 int class_idx;
1869 enum fullness_group fullness;
1870 struct zspage *zspage;
1871 struct address_space *mapping;
1872
1873
1874
1875
1876
1877 VM_BUG_ON_PAGE(!PageMovable(page), page);
1878 VM_BUG_ON_PAGE(PageIsolated(page), page);
1879
1880 zspage = get_zspage(page);
1881
1882
1883
1884
1885
1886
1887 get_zspage_mapping(zspage, &class_idx, &fullness);
1888 mapping = page_mapping(page);
1889 pool = mapping->private_data;
1890 class = pool->size_class[class_idx];
1891
1892 spin_lock(&class->lock);
1893 if (get_zspage_inuse(zspage) == 0) {
1894 spin_unlock(&class->lock);
1895 return false;
1896 }
1897
1898
1899 if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1900 spin_unlock(&class->lock);
1901 return false;
1902 }
1903
1904
1905
1906
1907
1908 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1909 get_zspage_mapping(zspage, &class_idx, &fullness);
1910 atomic_long_inc(&pool->isolated_pages);
1911 remove_zspage(class, zspage, fullness);
1912 }
1913
1914 inc_zspage_isolation(zspage);
1915 spin_unlock(&class->lock);
1916
1917 return true;
1918}
1919
1920static int zs_page_migrate(struct address_space *mapping, struct page *newpage,
1921 struct page *page, enum migrate_mode mode)
1922{
1923 struct zs_pool *pool;
1924 struct size_class *class;
1925 int class_idx;
1926 enum fullness_group fullness;
1927 struct zspage *zspage;
1928 struct page *dummy;
1929 void *s_addr, *d_addr, *addr;
1930 int offset, pos;
1931 unsigned long handle, head;
1932 unsigned long old_obj, new_obj;
1933 unsigned int obj_idx;
1934 int ret = -EAGAIN;
1935
1936
1937
1938
1939
1940
1941 if (mode == MIGRATE_SYNC_NO_COPY)
1942 return -EINVAL;
1943
1944 VM_BUG_ON_PAGE(!PageMovable(page), page);
1945 VM_BUG_ON_PAGE(!PageIsolated(page), page);
1946
1947 zspage = get_zspage(page);
1948
1949
1950 migrate_write_lock(zspage);
1951 get_zspage_mapping(zspage, &class_idx, &fullness);
1952 pool = mapping->private_data;
1953 class = pool->size_class[class_idx];
1954 offset = get_first_obj_offset(page);
1955
1956 spin_lock(&class->lock);
1957 if (!get_zspage_inuse(zspage)) {
1958
1959
1960
1961
1962 offset = PAGE_SIZE;
1963 }
1964
1965 pos = offset;
1966 s_addr = kmap_atomic(page);
1967 while (pos < PAGE_SIZE) {
1968 head = obj_to_head(page, s_addr + pos);
1969 if (head & OBJ_ALLOCATED_TAG) {
1970 handle = head & ~OBJ_ALLOCATED_TAG;
1971 if (!trypin_tag(handle))
1972 goto unpin_objects;
1973 }
1974 pos += class->size;
1975 }
1976
1977
1978
1979
1980 d_addr = kmap_atomic(newpage);
1981 memcpy(d_addr, s_addr, PAGE_SIZE);
1982 kunmap_atomic(d_addr);
1983
1984 for (addr = s_addr + offset; addr < s_addr + pos;
1985 addr += class->size) {
1986 head = obj_to_head(page, addr);
1987 if (head & OBJ_ALLOCATED_TAG) {
1988 handle = head & ~OBJ_ALLOCATED_TAG;
1989 BUG_ON(!testpin_tag(handle));
1990
1991 old_obj = handle_to_obj(handle);
1992 obj_to_location(old_obj, &dummy, &obj_idx);
1993 new_obj = (unsigned long)location_to_obj(newpage,
1994 obj_idx);
1995 new_obj |= BIT(HANDLE_PIN_BIT);
1996 record_obj(handle, new_obj);
1997 }
1998 }
1999
2000 replace_sub_page(class, zspage, newpage, page);
2001 get_page(newpage);
2002
2003 dec_zspage_isolation(zspage);
2004
2005
2006
2007
2008
2009 if (!is_zspage_isolated(zspage)) {
2010
2011
2012
2013
2014
2015
2016 putback_zspage_deferred(pool, class, zspage);
2017 zs_pool_dec_isolated(pool);
2018 }
2019
2020 if (page_zone(newpage) != page_zone(page)) {
2021 dec_zone_page_state(page, NR_ZSPAGES);
2022 inc_zone_page_state(newpage, NR_ZSPAGES);
2023 }
2024
2025 reset_page(page);
2026 put_page(page);
2027 page = newpage;
2028
2029 ret = MIGRATEPAGE_SUCCESS;
2030unpin_objects:
2031 for (addr = s_addr + offset; addr < s_addr + pos;
2032 addr += class->size) {
2033 head = obj_to_head(page, addr);
2034 if (head & OBJ_ALLOCATED_TAG) {
2035 handle = head & ~OBJ_ALLOCATED_TAG;
2036 BUG_ON(!testpin_tag(handle));
2037 unpin_tag(handle);
2038 }
2039 }
2040 kunmap_atomic(s_addr);
2041 spin_unlock(&class->lock);
2042 migrate_write_unlock(zspage);
2043
2044 return ret;
2045}
2046
2047static void zs_page_putback(struct page *page)
2048{
2049 struct zs_pool *pool;
2050 struct size_class *class;
2051 int class_idx;
2052 enum fullness_group fg;
2053 struct address_space *mapping;
2054 struct zspage *zspage;
2055
2056 VM_BUG_ON_PAGE(!PageMovable(page), page);
2057 VM_BUG_ON_PAGE(!PageIsolated(page), page);
2058
2059 zspage = get_zspage(page);
2060 get_zspage_mapping(zspage, &class_idx, &fg);
2061 mapping = page_mapping(page);
2062 pool = mapping->private_data;
2063 class = pool->size_class[class_idx];
2064
2065 spin_lock(&class->lock);
2066 dec_zspage_isolation(zspage);
2067 if (!is_zspage_isolated(zspage)) {
2068
2069
2070
2071
2072 putback_zspage_deferred(pool, class, zspage);
2073 zs_pool_dec_isolated(pool);
2074 }
2075 spin_unlock(&class->lock);
2076}
2077
2078static const struct address_space_operations zsmalloc_aops = {
2079 .isolate_page = zs_page_isolate,
2080 .migratepage = zs_page_migrate,
2081 .putback_page = zs_page_putback,
2082};
2083
2084static int zs_register_migration(struct zs_pool *pool)
2085{
2086 pool->inode = alloc_anon_inode(zsmalloc_mnt->mnt_sb);
2087 if (IS_ERR(pool->inode)) {
2088 pool->inode = NULL;
2089 return 1;
2090 }
2091
2092 pool->inode->i_mapping->private_data = pool;
2093 pool->inode->i_mapping->a_ops = &zsmalloc_aops;
2094 return 0;
2095}
2096
2097static bool pool_isolated_are_drained(struct zs_pool *pool)
2098{
2099 return atomic_long_read(&pool->isolated_pages) == 0;
2100}
2101
2102
2103static void wait_for_isolated_drain(struct zs_pool *pool)
2104{
2105
2106
2107
2108
2109
2110
2111
2112 wait_event(pool->migration_wait,
2113 pool_isolated_are_drained(pool));
2114}
2115
2116static void zs_unregister_migration(struct zs_pool *pool)
2117{
2118 pool->destroying = true;
2119
2120
2121
2122
2123
2124
2125 smp_mb();
2126 wait_for_isolated_drain(pool);
2127 flush_work(&pool->free_work);
2128 iput(pool->inode);
2129}
2130
2131
2132
2133
2134
2135static void async_free_zspage(struct work_struct *work)
2136{
2137 int i;
2138 struct size_class *class;
2139 unsigned int class_idx;
2140 enum fullness_group fullness;
2141 struct zspage *zspage, *tmp;
2142 LIST_HEAD(free_pages);
2143 struct zs_pool *pool = container_of(work, struct zs_pool,
2144 free_work);
2145
2146 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
2147 class = pool->size_class[i];
2148 if (class->index != i)
2149 continue;
2150
2151 spin_lock(&class->lock);
2152 list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages);
2153 spin_unlock(&class->lock);
2154 }
2155
2156
2157 list_for_each_entry_safe(zspage, tmp, &free_pages, list) {
2158 list_del(&zspage->list);
2159 lock_zspage(zspage);
2160
2161 get_zspage_mapping(zspage, &class_idx, &fullness);
2162 VM_BUG_ON(fullness != ZS_EMPTY);
2163 class = pool->size_class[class_idx];
2164 spin_lock(&class->lock);
2165 __free_zspage(pool, class, zspage);
2166 spin_unlock(&class->lock);
2167 }
2168};
2169
2170static void kick_deferred_free(struct zs_pool *pool)
2171{
2172 schedule_work(&pool->free_work);
2173}
2174
2175static void init_deferred_free(struct zs_pool *pool)
2176{
2177 INIT_WORK(&pool->free_work, async_free_zspage);
2178}
2179
2180static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)
2181{
2182 struct page *page = get_first_page(zspage);
2183
2184 do {
2185 WARN_ON(!trylock_page(page));
2186 __SetPageMovable(page, pool->inode->i_mapping);
2187 unlock_page(page);
2188 } while ((page = get_next_page(page)) != NULL);
2189}
2190#endif
2191
2192
2193
2194
2195
2196
2197static unsigned long zs_can_compact(struct size_class *class)
2198{
2199 unsigned long obj_wasted;
2200 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
2201 unsigned long obj_used = zs_stat_get(class, OBJ_USED);
2202
2203 if (obj_allocated <= obj_used)
2204 return 0;
2205
2206 obj_wasted = obj_allocated - obj_used;
2207 obj_wasted /= class->objs_per_zspage;
2208
2209 return obj_wasted * class->pages_per_zspage;
2210}
2211
2212static unsigned long __zs_compact(struct zs_pool *pool,
2213 struct size_class *class)
2214{
2215 struct zs_compact_control cc;
2216 struct zspage *src_zspage;
2217 struct zspage *dst_zspage = NULL;
2218 unsigned long pages_freed = 0;
2219
2220 spin_lock(&class->lock);
2221 while ((src_zspage = isolate_zspage(class, true))) {
2222
2223 if (!zs_can_compact(class))
2224 break;
2225
2226 cc.obj_idx = 0;
2227 cc.s_page = get_first_page(src_zspage);
2228
2229 while ((dst_zspage = isolate_zspage(class, false))) {
2230 cc.d_page = get_first_page(dst_zspage);
2231
2232
2233
2234
2235 if (!migrate_zspage(pool, class, &cc))
2236 break;
2237
2238 putback_zspage(class, dst_zspage);
2239 }
2240
2241
2242 if (dst_zspage == NULL)
2243 break;
2244
2245 putback_zspage(class, dst_zspage);
2246 if (putback_zspage(class, src_zspage) == ZS_EMPTY) {
2247 free_zspage(pool, class, src_zspage);
2248 pages_freed += class->pages_per_zspage;
2249 }
2250 spin_unlock(&class->lock);
2251 cond_resched();
2252 spin_lock(&class->lock);
2253 }
2254
2255 if (src_zspage)
2256 putback_zspage(class, src_zspage);
2257
2258 spin_unlock(&class->lock);
2259
2260 return pages_freed;
2261}
2262
2263unsigned long zs_compact(struct zs_pool *pool)
2264{
2265 int i;
2266 struct size_class *class;
2267 unsigned long pages_freed = 0;
2268
2269 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2270 class = pool->size_class[i];
2271 if (!class)
2272 continue;
2273 if (class->index != i)
2274 continue;
2275 pages_freed += __zs_compact(pool, class);
2276 }
2277 atomic_long_add(pages_freed, &pool->stats.pages_compacted);
2278
2279 return pages_freed;
2280}
2281EXPORT_SYMBOL_GPL(zs_compact);
2282
2283void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats)
2284{
2285 memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats));
2286}
2287EXPORT_SYMBOL_GPL(zs_pool_stats);
2288
2289static unsigned long zs_shrinker_scan(struct shrinker *shrinker,
2290 struct shrink_control *sc)
2291{
2292 unsigned long pages_freed;
2293 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2294 shrinker);
2295
2296
2297
2298
2299
2300
2301 pages_freed = zs_compact(pool);
2302
2303 return pages_freed ? pages_freed : SHRINK_STOP;
2304}
2305
2306static unsigned long zs_shrinker_count(struct shrinker *shrinker,
2307 struct shrink_control *sc)
2308{
2309 int i;
2310 struct size_class *class;
2311 unsigned long pages_to_free = 0;
2312 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2313 shrinker);
2314
2315 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2316 class = pool->size_class[i];
2317 if (!class)
2318 continue;
2319 if (class->index != i)
2320 continue;
2321
2322 pages_to_free += zs_can_compact(class);
2323 }
2324
2325 return pages_to_free;
2326}
2327
2328static void zs_unregister_shrinker(struct zs_pool *pool)
2329{
2330 unregister_shrinker(&pool->shrinker);
2331}
2332
2333static int zs_register_shrinker(struct zs_pool *pool)
2334{
2335 pool->shrinker.scan_objects = zs_shrinker_scan;
2336 pool->shrinker.count_objects = zs_shrinker_count;
2337 pool->shrinker.batch = 0;
2338 pool->shrinker.seeks = DEFAULT_SEEKS;
2339
2340 return register_shrinker(&pool->shrinker);
2341}
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353struct zs_pool *zs_create_pool(const char *name)
2354{
2355 int i;
2356 struct zs_pool *pool;
2357 struct size_class *prev_class = NULL;
2358
2359 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2360 if (!pool)
2361 return NULL;
2362
2363 init_deferred_free(pool);
2364
2365 pool->name = kstrdup(name, GFP_KERNEL);
2366 if (!pool->name)
2367 goto err;
2368
2369#ifdef CONFIG_COMPACTION
2370 init_waitqueue_head(&pool->migration_wait);
2371#endif
2372
2373 if (create_cache(pool))
2374 goto err;
2375
2376
2377
2378
2379
2380 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2381 int size;
2382 int pages_per_zspage;
2383 int objs_per_zspage;
2384 struct size_class *class;
2385 int fullness = 0;
2386
2387 size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA;
2388 if (size > ZS_MAX_ALLOC_SIZE)
2389 size = ZS_MAX_ALLOC_SIZE;
2390 pages_per_zspage = get_pages_per_zspage(size);
2391 objs_per_zspage = pages_per_zspage * PAGE_SIZE / size;
2392
2393
2394
2395
2396
2397
2398
2399 if (pages_per_zspage != 1 && objs_per_zspage != 1 &&
2400 !huge_class_size) {
2401 huge_class_size = size;
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411 huge_class_size -= (ZS_HANDLE_SIZE - 1);
2412 }
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423 if (prev_class) {
2424 if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) {
2425 pool->size_class[i] = prev_class;
2426 continue;
2427 }
2428 }
2429
2430 class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
2431 if (!class)
2432 goto err;
2433
2434 class->size = size;
2435 class->index = i;
2436 class->pages_per_zspage = pages_per_zspage;
2437 class->objs_per_zspage = objs_per_zspage;
2438 spin_lock_init(&class->lock);
2439 pool->size_class[i] = class;
2440 for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS;
2441 fullness++)
2442 INIT_LIST_HEAD(&class->fullness_list[fullness]);
2443
2444 prev_class = class;
2445 }
2446
2447
2448 zs_pool_stat_create(pool, name);
2449
2450 if (zs_register_migration(pool))
2451 goto err;
2452
2453
2454
2455
2456
2457
2458
2459 zs_register_shrinker(pool);
2460
2461 return pool;
2462
2463err:
2464 zs_destroy_pool(pool);
2465 return NULL;
2466}
2467EXPORT_SYMBOL_GPL(zs_create_pool);
2468
2469void zs_destroy_pool(struct zs_pool *pool)
2470{
2471 int i;
2472
2473 zs_unregister_shrinker(pool);
2474 zs_unregister_migration(pool);
2475 zs_pool_stat_destroy(pool);
2476
2477 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
2478 int fg;
2479 struct size_class *class = pool->size_class[i];
2480
2481 if (!class)
2482 continue;
2483
2484 if (class->index != i)
2485 continue;
2486
2487 for (fg = ZS_EMPTY; fg < NR_ZS_FULLNESS; fg++) {
2488 if (!list_empty(&class->fullness_list[fg])) {
2489 pr_info("Freeing non-empty class with size %db, fullness group %d\n",
2490 class->size, fg);
2491 }
2492 }
2493 kfree(class);
2494 }
2495
2496 destroy_cache(pool);
2497 kfree(pool->name);
2498 kfree(pool);
2499}
2500EXPORT_SYMBOL_GPL(zs_destroy_pool);
2501
2502static int __init zs_init(void)
2503{
2504 int ret;
2505
2506 ret = zsmalloc_mount();
2507 if (ret)
2508 goto out;
2509
2510 ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare",
2511 zs_cpu_prepare, zs_cpu_dead);
2512 if (ret)
2513 goto hp_setup_fail;
2514
2515#ifdef CONFIG_ZPOOL
2516 zpool_register_driver(&zs_zpool_driver);
2517#endif
2518
2519 zs_stat_init();
2520
2521 return 0;
2522
2523hp_setup_fail:
2524 zsmalloc_unmount();
2525out:
2526 return ret;
2527}
2528
2529static void __exit zs_exit(void)
2530{
2531#ifdef CONFIG_ZPOOL
2532 zpool_unregister_driver(&zs_zpool_driver);
2533#endif
2534 zsmalloc_unmount();
2535 cpuhp_remove_state(CPUHP_MM_ZS_PREPARE);
2536
2537 zs_stat_exit();
2538}
2539
2540module_init(zs_init);
2541module_exit(zs_exit);
2542
2543MODULE_LICENSE("Dual BSD/GPL");
2544MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
2545