1
2
3
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/init.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
9#include <linux/bootmem.h>
10#include <linux/gcd.h>
11#include <linux/lcm.h>
12#include <linux/jiffies.h>
13#include <linux/gfp.h>
14
15#include "blk.h"
16
17unsigned long blk_max_low_pfn;
18EXPORT_SYMBOL(blk_max_low_pfn);
19
20unsigned long blk_max_pfn;
21
22
23
24
25
26
27
28
29
30
31
32
33void blk_queue_prep_rq(struct request_queue *q, prep_rq_fn *pfn)
34{
35 q->prep_rq_fn = pfn;
36}
37EXPORT_SYMBOL(blk_queue_prep_rq);
38
39
40
41
42
43
44
45
46
47
48
49
50void blk_queue_unprep_rq(struct request_queue *q, unprep_rq_fn *ufn)
51{
52 q->unprep_rq_fn = ufn;
53}
54EXPORT_SYMBOL(blk_queue_unprep_rq);
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72void blk_queue_merge_bvec(struct request_queue *q, merge_bvec_fn *mbfn)
73{
74 q->merge_bvec_fn = mbfn;
75}
76EXPORT_SYMBOL(blk_queue_merge_bvec);
77
78void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn)
79{
80 q->softirq_done_fn = fn;
81}
82EXPORT_SYMBOL(blk_queue_softirq_done);
83
84void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)
85{
86 q->rq_timeout = timeout;
87}
88EXPORT_SYMBOL_GPL(blk_queue_rq_timeout);
89
90void blk_queue_rq_timed_out(struct request_queue *q, rq_timed_out_fn *fn)
91{
92 q->rq_timed_out_fn = fn;
93}
94EXPORT_SYMBOL_GPL(blk_queue_rq_timed_out);
95
96void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn)
97{
98 q->lld_busy_fn = fn;
99}
100EXPORT_SYMBOL_GPL(blk_queue_lld_busy);
101
102
103
104
105
106
107
108
109
110
111void blk_set_default_limits(struct queue_limits *lim)
112{
113 lim->max_segments = BLK_MAX_SEGMENTS;
114 lim->max_integrity_segments = 0;
115 lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK;
116 lim->max_segment_size = BLK_MAX_SEGMENT_SIZE;
117 lim->max_sectors = BLK_DEF_MAX_SECTORS;
118 lim->max_hw_sectors = INT_MAX;
119 lim->max_discard_sectors = 0;
120 lim->discard_granularity = 0;
121 lim->discard_alignment = 0;
122 lim->discard_misaligned = 0;
123 lim->discard_zeroes_data = -1;
124 lim->logical_block_size = lim->physical_block_size = lim->io_min = 512;
125 lim->bounce_pfn = (unsigned long)(BLK_BOUNCE_ANY >> PAGE_SHIFT);
126 lim->alignment_offset = 0;
127 lim->io_opt = 0;
128 lim->misaligned = 0;
129 lim->cluster = 1;
130}
131EXPORT_SYMBOL(blk_set_default_limits);
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155void blk_queue_make_request(struct request_queue *q, make_request_fn *mfn)
156{
157
158
159
160 q->nr_requests = BLKDEV_MAX_RQ;
161
162 q->make_request_fn = mfn;
163 blk_queue_dma_alignment(q, 511);
164 blk_queue_congestion_threshold(q);
165 q->nr_batching = BLK_BATCH_REQ;
166
167 blk_set_default_limits(&q->limits);
168 blk_queue_max_hw_sectors(q, BLK_SAFE_MAX_SECTORS);
169
170
171
172
173 blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
174}
175EXPORT_SYMBOL(blk_queue_make_request);
176
177
178
179
180
181
182
183
184
185
186
187
188void blk_queue_bounce_limit(struct request_queue *q, u64 dma_mask)
189{
190 unsigned long b_pfn = dma_mask >> PAGE_SHIFT;
191 int dma = 0;
192
193 q->bounce_gfp = GFP_NOIO;
194#if BITS_PER_LONG == 64
195
196
197
198
199
200 if (b_pfn < (min_t(u64, 0xffffffffUL, BLK_BOUNCE_HIGH) >> PAGE_SHIFT))
201 dma = 1;
202 q->limits.bounce_pfn = max(max_low_pfn, b_pfn);
203#else
204 if (b_pfn < blk_max_low_pfn)
205 dma = 1;
206 q->limits.bounce_pfn = b_pfn;
207#endif
208 if (dma) {
209 init_emergency_isa_pool();
210 q->bounce_gfp = GFP_NOIO | GFP_DMA;
211 q->limits.bounce_pfn = b_pfn;
212 }
213}
214EXPORT_SYMBOL(blk_queue_bounce_limit);
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232void blk_limits_max_hw_sectors(struct queue_limits *limits, unsigned int max_hw_sectors)
233{
234 if ((max_hw_sectors << 9) < PAGE_CACHE_SIZE) {
235 max_hw_sectors = 1 << (PAGE_CACHE_SHIFT - 9);
236 printk(KERN_INFO "%s: set to minimum %d\n",
237 __func__, max_hw_sectors);
238 }
239
240 limits->max_hw_sectors = max_hw_sectors;
241 limits->max_sectors = min_t(unsigned int, max_hw_sectors,
242 BLK_DEF_MAX_SECTORS);
243}
244EXPORT_SYMBOL(blk_limits_max_hw_sectors);
245
246
247
248
249
250
251
252
253
254void blk_queue_max_hw_sectors(struct request_queue *q, unsigned int max_hw_sectors)
255{
256 blk_limits_max_hw_sectors(&q->limits, max_hw_sectors);
257}
258EXPORT_SYMBOL(blk_queue_max_hw_sectors);
259
260
261
262
263
264
265void blk_queue_max_discard_sectors(struct request_queue *q,
266 unsigned int max_discard_sectors)
267{
268 q->limits.max_discard_sectors = max_discard_sectors;
269}
270EXPORT_SYMBOL(blk_queue_max_discard_sectors);
271
272
273
274
275
276
277
278
279
280
281void blk_queue_max_segments(struct request_queue *q, unsigned short max_segments)
282{
283 if (!max_segments) {
284 max_segments = 1;
285 printk(KERN_INFO "%s: set to minimum %d\n",
286 __func__, max_segments);
287 }
288
289 q->limits.max_segments = max_segments;
290}
291EXPORT_SYMBOL(blk_queue_max_segments);
292
293
294
295
296
297
298
299
300
301
302void blk_queue_max_segment_size(struct request_queue *q, unsigned int max_size)
303{
304 if (max_size < PAGE_CACHE_SIZE) {
305 max_size = PAGE_CACHE_SIZE;
306 printk(KERN_INFO "%s: set to minimum %d\n",
307 __func__, max_size);
308 }
309
310 q->limits.max_segment_size = max_size;
311}
312EXPORT_SYMBOL(blk_queue_max_segment_size);
313
314
315
316
317
318
319
320
321
322
323
324void blk_queue_logical_block_size(struct request_queue *q, unsigned short size)
325{
326 q->limits.logical_block_size = size;
327
328 if (q->limits.physical_block_size < size)
329 q->limits.physical_block_size = size;
330
331 if (q->limits.io_min < q->limits.physical_block_size)
332 q->limits.io_min = q->limits.physical_block_size;
333}
334EXPORT_SYMBOL(blk_queue_logical_block_size);
335
336
337
338
339
340
341
342
343
344
345
346void blk_queue_physical_block_size(struct request_queue *q, unsigned int size)
347{
348 q->limits.physical_block_size = size;
349
350 if (q->limits.physical_block_size < q->limits.logical_block_size)
351 q->limits.physical_block_size = q->limits.logical_block_size;
352
353 if (q->limits.io_min < q->limits.physical_block_size)
354 q->limits.io_min = q->limits.physical_block_size;
355}
356EXPORT_SYMBOL(blk_queue_physical_block_size);
357
358
359
360
361
362
363
364
365
366
367
368
369void blk_queue_alignment_offset(struct request_queue *q, unsigned int offset)
370{
371 q->limits.alignment_offset =
372 offset & (q->limits.physical_block_size - 1);
373 q->limits.misaligned = 0;
374}
375EXPORT_SYMBOL(blk_queue_alignment_offset);
376
377
378
379
380
381
382
383
384
385
386
387
388void blk_limits_io_min(struct queue_limits *limits, unsigned int min)
389{
390 limits->io_min = min;
391
392 if (limits->io_min < limits->logical_block_size)
393 limits->io_min = limits->logical_block_size;
394
395 if (limits->io_min < limits->physical_block_size)
396 limits->io_min = limits->physical_block_size;
397}
398EXPORT_SYMBOL(blk_limits_io_min);
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414void blk_queue_io_min(struct request_queue *q, unsigned int min)
415{
416 blk_limits_io_min(&q->limits, min);
417}
418EXPORT_SYMBOL(blk_queue_io_min);
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt)
434{
435 limits->io_opt = opt;
436}
437EXPORT_SYMBOL(blk_limits_io_opt);
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452void blk_queue_io_opt(struct request_queue *q, unsigned int opt)
453{
454 blk_limits_io_opt(&q->limits, opt);
455}
456EXPORT_SYMBOL(blk_queue_io_opt);
457
458
459
460
461
462
463void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b)
464{
465 blk_stack_limits(&t->limits, &b->limits, 0);
466}
467EXPORT_SYMBOL(blk_queue_stack_limits);
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
491 sector_t start)
492{
493 unsigned int top, bottom, alignment, ret = 0;
494
495 t->max_sectors = min_not_zero(t->max_sectors, b->max_sectors);
496 t->max_hw_sectors = min_not_zero(t->max_hw_sectors, b->max_hw_sectors);
497 t->bounce_pfn = min_not_zero(t->bounce_pfn, b->bounce_pfn);
498
499 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask,
500 b->seg_boundary_mask);
501
502 t->max_segments = min_not_zero(t->max_segments, b->max_segments);
503 t->max_integrity_segments = min_not_zero(t->max_integrity_segments,
504 b->max_integrity_segments);
505
506 t->max_segment_size = min_not_zero(t->max_segment_size,
507 b->max_segment_size);
508
509 t->misaligned |= b->misaligned;
510
511 alignment = queue_limit_alignment_offset(b, start);
512
513
514
515
516 if (t->alignment_offset != alignment) {
517
518 top = max(t->physical_block_size, t->io_min)
519 + t->alignment_offset;
520 bottom = max(b->physical_block_size, b->io_min) + alignment;
521
522
523 if (max(top, bottom) & (min(top, bottom) - 1)) {
524 t->misaligned = 1;
525 ret = -1;
526 }
527 }
528
529 t->logical_block_size = max(t->logical_block_size,
530 b->logical_block_size);
531
532 t->physical_block_size = max(t->physical_block_size,
533 b->physical_block_size);
534
535 t->io_min = max(t->io_min, b->io_min);
536 t->io_opt = lcm(t->io_opt, b->io_opt);
537
538 t->cluster &= b->cluster;
539 t->discard_zeroes_data &= b->discard_zeroes_data;
540
541
542 if (t->physical_block_size & (t->logical_block_size - 1)) {
543 t->physical_block_size = t->logical_block_size;
544 t->misaligned = 1;
545 ret = -1;
546 }
547
548
549 if (t->io_min & (t->physical_block_size - 1)) {
550 t->io_min = t->physical_block_size;
551 t->misaligned = 1;
552 ret = -1;
553 }
554
555
556 if (t->io_opt & (t->physical_block_size - 1)) {
557 t->io_opt = 0;
558 t->misaligned = 1;
559 ret = -1;
560 }
561
562
563 t->alignment_offset = lcm(t->alignment_offset, alignment)
564 & (max(t->physical_block_size, t->io_min) - 1);
565
566
567 if (t->alignment_offset & (t->logical_block_size - 1)) {
568 t->misaligned = 1;
569 ret = -1;
570 }
571
572
573 if (b->discard_granularity) {
574 alignment = queue_limit_discard_alignment(b, start);
575
576 if (t->discard_granularity != 0 &&
577 t->discard_alignment != alignment) {
578 top = t->discard_granularity + t->discard_alignment;
579 bottom = b->discard_granularity + alignment;
580
581
582 if (max(top, bottom) & (min(top, bottom) - 1))
583 t->discard_misaligned = 1;
584 }
585
586 t->max_discard_sectors = min_not_zero(t->max_discard_sectors,
587 b->max_discard_sectors);
588 t->discard_granularity = max(t->discard_granularity,
589 b->discard_granularity);
590 t->discard_alignment = lcm(t->discard_alignment, alignment) &
591 (t->discard_granularity - 1);
592 }
593
594 return ret;
595}
596EXPORT_SYMBOL(blk_stack_limits);
597
598
599
600
601
602
603
604
605
606
607
608
609int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
610 sector_t start)
611{
612 struct request_queue *bq = bdev_get_queue(bdev);
613
614 start += get_start_sect(bdev);
615
616 return blk_stack_limits(t, &bq->limits, start);
617}
618EXPORT_SYMBOL(bdev_stack_limits);
619
620
621
622
623
624
625
626
627
628
629
630void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
631 sector_t offset)
632{
633 struct request_queue *t = disk->queue;
634
635 if (bdev_stack_limits(&t->limits, bdev, offset >> 9) < 0) {
636 char top[BDEVNAME_SIZE], bottom[BDEVNAME_SIZE];
637
638 disk_name(disk, 0, top);
639 bdevname(bdev, bottom);
640
641 printk(KERN_NOTICE "%s: Warning: Device %s is misaligned\n",
642 top, bottom);
643 }
644}
645EXPORT_SYMBOL(disk_stack_limits);
646
647
648
649
650
651
652
653
654
655
656
657void blk_queue_dma_pad(struct request_queue *q, unsigned int mask)
658{
659 q->dma_pad_mask = mask;
660}
661EXPORT_SYMBOL(blk_queue_dma_pad);
662
663
664
665
666
667
668
669
670
671
672
673void blk_queue_update_dma_pad(struct request_queue *q, unsigned int mask)
674{
675 if (mask > q->dma_pad_mask)
676 q->dma_pad_mask = mask;
677}
678EXPORT_SYMBOL(blk_queue_update_dma_pad);
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701int blk_queue_dma_drain(struct request_queue *q,
702 dma_drain_needed_fn *dma_drain_needed,
703 void *buf, unsigned int size)
704{
705 if (queue_max_segments(q) < 2)
706 return -EINVAL;
707
708 blk_queue_max_segments(q, queue_max_segments(q) - 1);
709 q->dma_drain_needed = dma_drain_needed;
710 q->dma_drain_buffer = buf;
711 q->dma_drain_size = size;
712
713 return 0;
714}
715EXPORT_SYMBOL_GPL(blk_queue_dma_drain);
716
717
718
719
720
721
722void blk_queue_segment_boundary(struct request_queue *q, unsigned long mask)
723{
724 if (mask < PAGE_CACHE_SIZE - 1) {
725 mask = PAGE_CACHE_SIZE - 1;
726 printk(KERN_INFO "%s: set to minimum %lx\n",
727 __func__, mask);
728 }
729
730 q->limits.seg_boundary_mask = mask;
731}
732EXPORT_SYMBOL(blk_queue_segment_boundary);
733
734
735
736
737
738
739
740
741
742
743
744void blk_queue_dma_alignment(struct request_queue *q, int mask)
745{
746 q->dma_alignment = mask;
747}
748EXPORT_SYMBOL(blk_queue_dma_alignment);
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764void blk_queue_update_dma_alignment(struct request_queue *q, int mask)
765{
766 BUG_ON(mask > PAGE_SIZE);
767
768 if (mask > q->dma_alignment)
769 q->dma_alignment = mask;
770}
771EXPORT_SYMBOL(blk_queue_update_dma_alignment);
772
773
774
775
776
777
778
779
780
781
782void blk_queue_flush(struct request_queue *q, unsigned int flush)
783{
784 WARN_ON_ONCE(flush & ~(REQ_FLUSH | REQ_FUA));
785
786 if (WARN_ON_ONCE(!(flush & REQ_FLUSH) && (flush & REQ_FUA)))
787 flush &= ~REQ_FUA;
788
789 q->flush_flags = flush & (REQ_FLUSH | REQ_FUA);
790}
791EXPORT_SYMBOL_GPL(blk_queue_flush);
792
793static int __init blk_settings_init(void)
794{
795 blk_max_low_pfn = max_low_pfn - 1;
796 blk_max_pfn = max_pfn - 1;
797 return 0;
798}
799subsys_initcall(blk_settings_init);
800