1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#ifndef TEST
30#include <linux/slab.h>
31#include <linux/init.h>
32#include <linux/module.h>
33#endif
34#include <linux/err.h>
35#include <linux/string.h>
36#include <linux/idr.h>
37
38static struct kmem_cache *idr_layer_cache;
39
40static struct idr_layer *get_from_free_list(struct idr *idp)
41{
42 struct idr_layer *p;
43 unsigned long flags;
44
45 spin_lock_irqsave(&idp->lock, flags);
46 if ((p = idp->id_free)) {
47 idp->id_free = p->ary[0];
48 idp->id_free_cnt--;
49 p->ary[0] = NULL;
50 }
51 spin_unlock_irqrestore(&idp->lock, flags);
52 return(p);
53}
54
55static void idr_layer_rcu_free(struct rcu_head *head)
56{
57 struct idr_layer *layer;
58
59 layer = container_of(head, struct idr_layer, rcu_head);
60 kmem_cache_free(idr_layer_cache, layer);
61}
62
63static inline void free_layer(struct idr_layer *p)
64{
65 call_rcu(&p->rcu_head, idr_layer_rcu_free);
66}
67
68
69static void __move_to_free_list(struct idr *idp, struct idr_layer *p)
70{
71 p->ary[0] = idp->id_free;
72 idp->id_free = p;
73 idp->id_free_cnt++;
74}
75
76static void move_to_free_list(struct idr *idp, struct idr_layer *p)
77{
78 unsigned long flags;
79
80
81
82
83 spin_lock_irqsave(&idp->lock, flags);
84 __move_to_free_list(idp, p);
85 spin_unlock_irqrestore(&idp->lock, flags);
86}
87
88static void idr_mark_full(struct idr_layer **pa, int id)
89{
90 struct idr_layer *p = pa[0];
91 int l = 0;
92
93 __set_bit(id & IDR_MASK, &p->bitmap);
94
95
96
97
98
99
100 while (p->bitmap == IDR_FULL) {
101 if (!(p = pa[++l]))
102 break;
103 id = id >> IDR_BITS;
104 __set_bit((id & IDR_MASK), &p->bitmap);
105 }
106}
107
108
109
110
111
112
113
114
115
116
117
118
119
120int idr_pre_get(struct idr *idp, gfp_t gfp_mask)
121{
122 while (idp->id_free_cnt < IDR_FREE_MAX) {
123 struct idr_layer *new;
124 new = kmem_cache_zalloc(idr_layer_cache, gfp_mask);
125 if (new == NULL)
126 return (0);
127 move_to_free_list(idp, new);
128 }
129 return 1;
130}
131EXPORT_SYMBOL(idr_pre_get);
132
133static int sub_alloc(struct idr *idp, int *starting_id, struct idr_layer **pa)
134{
135 int n, m, sh;
136 struct idr_layer *p, *new;
137 int l, id, oid;
138 unsigned long bm;
139
140 id = *starting_id;
141 restart:
142 p = idp->top;
143 l = idp->layers;
144 pa[l--] = NULL;
145 while (1) {
146
147
148
149 n = (id >> (IDR_BITS*l)) & IDR_MASK;
150 bm = ~p->bitmap;
151 m = find_next_bit(&bm, IDR_SIZE, n);
152 if (m == IDR_SIZE) {
153
154 l++;
155 oid = id;
156 id = (id | ((1 << (IDR_BITS * l)) - 1)) + 1;
157
158
159 if (!(p = pa[l])) {
160 *starting_id = id;
161 return IDR_NEED_TO_GROW;
162 }
163
164
165
166
167 sh = IDR_BITS * (l + 1);
168 if (oid >> sh == id >> sh)
169 continue;
170 else
171 goto restart;
172 }
173 if (m != n) {
174 sh = IDR_BITS*l;
175 id = ((id >> sh) ^ n ^ m) << sh;
176 }
177 if ((id >= MAX_ID_BIT) || (id < 0))
178 return IDR_NOMORE_SPACE;
179 if (l == 0)
180 break;
181
182
183
184 if (!p->ary[m]) {
185 new = get_from_free_list(idp);
186 if (!new)
187 return -1;
188 new->layer = l-1;
189 rcu_assign_pointer(p->ary[m], new);
190 p->count++;
191 }
192 pa[l--] = p;
193 p = p->ary[m];
194 }
195
196 pa[l] = p;
197 return id;
198}
199
200static int idr_get_empty_slot(struct idr *idp, int starting_id,
201 struct idr_layer **pa)
202{
203 struct idr_layer *p, *new;
204 int layers, v, id;
205 unsigned long flags;
206
207 id = starting_id;
208build_up:
209 p = idp->top;
210 layers = idp->layers;
211 if (unlikely(!p)) {
212 if (!(p = get_from_free_list(idp)))
213 return -1;
214 p->layer = 0;
215 layers = 1;
216 }
217
218
219
220
221 while ((layers < (MAX_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) {
222 layers++;
223 if (!p->count) {
224
225
226
227
228 p->layer++;
229 continue;
230 }
231 if (!(new = get_from_free_list(idp))) {
232
233
234
235
236 spin_lock_irqsave(&idp->lock, flags);
237 for (new = p; p && p != idp->top; new = p) {
238 p = p->ary[0];
239 new->ary[0] = NULL;
240 new->bitmap = new->count = 0;
241 __move_to_free_list(idp, new);
242 }
243 spin_unlock_irqrestore(&idp->lock, flags);
244 return -1;
245 }
246 new->ary[0] = p;
247 new->count = 1;
248 new->layer = layers-1;
249 if (p->bitmap == IDR_FULL)
250 __set_bit(0, &new->bitmap);
251 p = new;
252 }
253 rcu_assign_pointer(idp->top, p);
254 idp->layers = layers;
255 v = sub_alloc(idp, &id, pa);
256 if (v == IDR_NEED_TO_GROW)
257 goto build_up;
258 return(v);
259}
260
261static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id)
262{
263 struct idr_layer *pa[MAX_LEVEL];
264 int id;
265
266 id = idr_get_empty_slot(idp, starting_id, pa);
267 if (id >= 0) {
268
269
270
271
272 rcu_assign_pointer(pa[0]->ary[id & IDR_MASK],
273 (struct idr_layer *)ptr);
274 pa[0]->count++;
275 idr_mark_full(pa, id);
276 }
277
278 return id;
279}
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id)
298{
299 int rv;
300
301 rv = idr_get_new_above_int(idp, ptr, starting_id);
302
303
304
305
306 if (rv < 0)
307 return _idr_rc_to_errno(rv);
308 *id = rv;
309 return 0;
310}
311EXPORT_SYMBOL(idr_get_new_above);
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328int idr_get_new(struct idr *idp, void *ptr, int *id)
329{
330 int rv;
331
332 rv = idr_get_new_above_int(idp, ptr, 0);
333
334
335
336
337 if (rv < 0)
338 return _idr_rc_to_errno(rv);
339 *id = rv;
340 return 0;
341}
342EXPORT_SYMBOL(idr_get_new);
343
344static void idr_remove_warning(int id)
345{
346 printk(KERN_WARNING
347 "idr_remove called for id=%d which is not allocated.\n", id);
348 dump_stack();
349}
350
351static void sub_remove(struct idr *idp, int shift, int id)
352{
353 struct idr_layer *p = idp->top;
354 struct idr_layer **pa[MAX_LEVEL];
355 struct idr_layer ***paa = &pa[0];
356 struct idr_layer *to_free;
357 int n;
358
359 *paa = NULL;
360 *++paa = &idp->top;
361
362 while ((shift > 0) && p) {
363 n = (id >> shift) & IDR_MASK;
364 __clear_bit(n, &p->bitmap);
365 *++paa = &p->ary[n];
366 p = p->ary[n];
367 shift -= IDR_BITS;
368 }
369 n = id & IDR_MASK;
370 if (likely(p != NULL && test_bit(n, &p->bitmap))){
371 __clear_bit(n, &p->bitmap);
372 rcu_assign_pointer(p->ary[n], NULL);
373 to_free = NULL;
374 while(*paa && ! --((**paa)->count)){
375 if (to_free)
376 free_layer(to_free);
377 to_free = **paa;
378 **paa-- = NULL;
379 }
380 if (!*paa)
381 idp->layers = 0;
382 if (to_free)
383 free_layer(to_free);
384 } else
385 idr_remove_warning(id);
386}
387
388
389
390
391
392
393void idr_remove(struct idr *idp, int id)
394{
395 struct idr_layer *p;
396 struct idr_layer *to_free;
397
398
399 id &= MAX_ID_MASK;
400
401 sub_remove(idp, (idp->layers - 1) * IDR_BITS, id);
402 if (idp->top && idp->top->count == 1 && (idp->layers > 1) &&
403 idp->top->ary[0]) {
404
405
406
407
408
409
410 to_free = idp->top;
411 p = idp->top->ary[0];
412 rcu_assign_pointer(idp->top, p);
413 --idp->layers;
414 to_free->bitmap = to_free->count = 0;
415 free_layer(to_free);
416 }
417 while (idp->id_free_cnt >= IDR_FREE_MAX) {
418 p = get_from_free_list(idp);
419
420
421
422
423
424 kmem_cache_free(idr_layer_cache, p);
425 }
426 return;
427}
428EXPORT_SYMBOL(idr_remove);
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443void idr_remove_all(struct idr *idp)
444{
445 int n, id, max;
446 struct idr_layer *p;
447 struct idr_layer *pa[MAX_LEVEL];
448 struct idr_layer **paa = &pa[0];
449
450 n = idp->layers * IDR_BITS;
451 p = idp->top;
452 max = 1 << n;
453
454 id = 0;
455 while (id < max) {
456 while (n > IDR_BITS && p) {
457 n -= IDR_BITS;
458 *paa++ = p;
459 p = p->ary[(id >> n) & IDR_MASK];
460 }
461
462 id += 1 << n;
463 while (n < fls(id)) {
464 if (p)
465 free_layer(p);
466 n += IDR_BITS;
467 p = *--paa;
468 }
469 }
470 rcu_assign_pointer(idp->top, NULL);
471 idp->layers = 0;
472}
473EXPORT_SYMBOL(idr_remove_all);
474
475
476
477
478
479void idr_destroy(struct idr *idp)
480{
481 while (idp->id_free_cnt) {
482 struct idr_layer *p = get_from_free_list(idp);
483 kmem_cache_free(idr_layer_cache, p);
484 }
485}
486EXPORT_SYMBOL(idr_destroy);
487
488
489
490
491
492
493
494
495
496
497
498
499
500void *idr_find(struct idr *idp, int id)
501{
502 int n;
503 struct idr_layer *p;
504
505 p = rcu_dereference(idp->top);
506 if (!p)
507 return NULL;
508 n = (p->layer+1) * IDR_BITS;
509
510
511 id &= MAX_ID_MASK;
512
513 if (id >= (1 << n))
514 return NULL;
515 BUG_ON(n == 0);
516
517 while (n > 0 && p) {
518 n -= IDR_BITS;
519 BUG_ON(n != p->layer*IDR_BITS);
520 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]);
521 }
522 return((void *)p);
523}
524EXPORT_SYMBOL(idr_find);
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544int idr_for_each(struct idr *idp,
545 int (*fn)(int id, void *p, void *data), void *data)
546{
547 int n, id, max, error = 0;
548 struct idr_layer *p;
549 struct idr_layer *pa[MAX_LEVEL];
550 struct idr_layer **paa = &pa[0];
551
552 n = idp->layers * IDR_BITS;
553 p = rcu_dereference(idp->top);
554 max = 1 << n;
555
556 id = 0;
557 while (id < max) {
558 while (n > 0 && p) {
559 n -= IDR_BITS;
560 *paa++ = p;
561 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]);
562 }
563
564 if (p) {
565 error = fn(id, (void *)p, data);
566 if (error)
567 break;
568 }
569
570 id += 1 << n;
571 while (n < fls(id)) {
572 n += IDR_BITS;
573 p = *--paa;
574 }
575 }
576
577 return error;
578}
579EXPORT_SYMBOL(idr_for_each);
580
581
582
583
584
585
586
587
588
589
590
591
592
593void *idr_replace(struct idr *idp, void *ptr, int id)
594{
595 int n;
596 struct idr_layer *p, *old_p;
597
598 p = idp->top;
599 if (!p)
600 return ERR_PTR(-EINVAL);
601
602 n = (p->layer+1) * IDR_BITS;
603
604 id &= MAX_ID_MASK;
605
606 if (id >= (1 << n))
607 return ERR_PTR(-EINVAL);
608
609 n -= IDR_BITS;
610 while ((n > 0) && p) {
611 p = p->ary[(id >> n) & IDR_MASK];
612 n -= IDR_BITS;
613 }
614
615 n = id & IDR_MASK;
616 if (unlikely(p == NULL || !test_bit(n, &p->bitmap)))
617 return ERR_PTR(-ENOENT);
618
619 old_p = p->ary[n];
620 rcu_assign_pointer(p->ary[n], ptr);
621
622 return old_p;
623}
624EXPORT_SYMBOL(idr_replace);
625
626void __init idr_init_cache(void)
627{
628 idr_layer_cache = kmem_cache_create("idr_layer_cache",
629 sizeof(struct idr_layer), 0, SLAB_PANIC, NULL);
630}
631
632
633
634
635
636
637
638
639void idr_init(struct idr *idp)
640{
641 memset(idp, 0, sizeof(struct idr));
642 spin_lock_init(&idp->lock);
643}
644EXPORT_SYMBOL(idr_init);
645
646
647
648
649
650
651
652
653
654
655
656
657
658static void free_bitmap(struct ida *ida, struct ida_bitmap *bitmap)
659{
660 unsigned long flags;
661
662 if (!ida->free_bitmap) {
663 spin_lock_irqsave(&ida->idr.lock, flags);
664 if (!ida->free_bitmap) {
665 ida->free_bitmap = bitmap;
666 bitmap = NULL;
667 }
668 spin_unlock_irqrestore(&ida->idr.lock, flags);
669 }
670
671 kfree(bitmap);
672}
673
674
675
676
677
678
679
680
681
682
683
684
685
686int ida_pre_get(struct ida *ida, gfp_t gfp_mask)
687{
688
689 if (!idr_pre_get(&ida->idr, gfp_mask))
690 return 0;
691
692
693 if (!ida->free_bitmap) {
694 struct ida_bitmap *bitmap;
695
696 bitmap = kmalloc(sizeof(struct ida_bitmap), gfp_mask);
697 if (!bitmap)
698 return 0;
699
700 free_bitmap(ida, bitmap);
701 }
702
703 return 1;
704}
705EXPORT_SYMBOL(ida_pre_get);
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
723{
724 struct idr_layer *pa[MAX_LEVEL];
725 struct ida_bitmap *bitmap;
726 unsigned long flags;
727 int idr_id = starting_id / IDA_BITMAP_BITS;
728 int offset = starting_id % IDA_BITMAP_BITS;
729 int t, id;
730
731 restart:
732
733 t = idr_get_empty_slot(&ida->idr, idr_id, pa);
734 if (t < 0)
735 return _idr_rc_to_errno(t);
736
737 if (t * IDA_BITMAP_BITS >= MAX_ID_BIT)
738 return -ENOSPC;
739
740 if (t != idr_id)
741 offset = 0;
742 idr_id = t;
743
744
745 bitmap = (void *)pa[0]->ary[idr_id & IDR_MASK];
746 if (!bitmap) {
747 spin_lock_irqsave(&ida->idr.lock, flags);
748 bitmap = ida->free_bitmap;
749 ida->free_bitmap = NULL;
750 spin_unlock_irqrestore(&ida->idr.lock, flags);
751
752 if (!bitmap)
753 return -EAGAIN;
754
755 memset(bitmap, 0, sizeof(struct ida_bitmap));
756 rcu_assign_pointer(pa[0]->ary[idr_id & IDR_MASK],
757 (void *)bitmap);
758 pa[0]->count++;
759 }
760
761
762 t = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, offset);
763 if (t == IDA_BITMAP_BITS) {
764
765 idr_id++;
766 offset = 0;
767 goto restart;
768 }
769
770 id = idr_id * IDA_BITMAP_BITS + t;
771 if (id >= MAX_ID_BIT)
772 return -ENOSPC;
773
774 __set_bit(t, bitmap->bitmap);
775 if (++bitmap->nr_busy == IDA_BITMAP_BITS)
776 idr_mark_full(pa, idr_id);
777
778 *p_id = id;
779
780
781
782
783
784
785 if (ida->idr.id_free_cnt || ida->free_bitmap) {
786 struct idr_layer *p = get_from_free_list(&ida->idr);
787 if (p)
788 kmem_cache_free(idr_layer_cache, p);
789 }
790
791 return 0;
792}
793EXPORT_SYMBOL(ida_get_new_above);
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808int ida_get_new(struct ida *ida, int *p_id)
809{
810 return ida_get_new_above(ida, 0, p_id);
811}
812EXPORT_SYMBOL(ida_get_new);
813
814
815
816
817
818
819void ida_remove(struct ida *ida, int id)
820{
821 struct idr_layer *p = ida->idr.top;
822 int shift = (ida->idr.layers - 1) * IDR_BITS;
823 int idr_id = id / IDA_BITMAP_BITS;
824 int offset = id % IDA_BITMAP_BITS;
825 int n;
826 struct ida_bitmap *bitmap;
827
828
829 while ((shift > 0) && p) {
830 n = (idr_id >> shift) & IDR_MASK;
831 __clear_bit(n, &p->bitmap);
832 p = p->ary[n];
833 shift -= IDR_BITS;
834 }
835
836 if (p == NULL)
837 goto err;
838
839 n = idr_id & IDR_MASK;
840 __clear_bit(n, &p->bitmap);
841
842 bitmap = (void *)p->ary[n];
843 if (!test_bit(offset, bitmap->bitmap))
844 goto err;
845
846
847 __clear_bit(offset, bitmap->bitmap);
848 if (--bitmap->nr_busy == 0) {
849 __set_bit(n, &p->bitmap);
850 idr_remove(&ida->idr, idr_id);
851 free_bitmap(ida, bitmap);
852 }
853
854 return;
855
856 err:
857 printk(KERN_WARNING
858 "ida_remove called for id=%d which is not allocated.\n", id);
859}
860EXPORT_SYMBOL(ida_remove);
861
862
863
864
865
866void ida_destroy(struct ida *ida)
867{
868 idr_destroy(&ida->idr);
869 kfree(ida->free_bitmap);
870}
871EXPORT_SYMBOL(ida_destroy);
872
873
874
875
876
877
878
879
880void ida_init(struct ida *ida)
881{
882 memset(ida, 0, sizeof(struct ida));
883 idr_init(&ida->idr);
884
885}
886EXPORT_SYMBOL(ida_init);
887