1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/spinlock.h>
19#include <linux/sched.h>
20#include <linux/fs.h>
21#include <linux/mm.h>
22#include <linux/writeback.h>
23#include <linux/blkdev.h>
24#include <linux/backing-dev.h>
25#include <linux/buffer_head.h>
26#include "internal.h"
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41static int writeback_acquire(struct backing_dev_info *bdi)
42{
43 return !test_and_set_bit(BDI_pdflush, &bdi->state);
44}
45
46
47
48
49
50
51
52int writeback_in_progress(struct backing_dev_info *bdi)
53{
54 return test_bit(BDI_pdflush, &bdi->state);
55}
56
57
58
59
60
61static void writeback_release(struct backing_dev_info *bdi)
62{
63 BUG_ON(!writeback_in_progress(bdi));
64 clear_bit(BDI_pdflush, &bdi->state);
65}
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94void __mark_inode_dirty(struct inode *inode, int flags)
95{
96 struct super_block *sb = inode->i_sb;
97
98
99
100
101
102 if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
103 if (sb->s_op->dirty_inode)
104 sb->s_op->dirty_inode(inode);
105 }
106
107
108
109
110
111 smp_mb();
112
113
114 if ((inode->i_state & flags) == flags)
115 return;
116
117 if (unlikely(block_dump)) {
118 struct dentry *dentry = NULL;
119 const char *name = "?";
120
121 if (!list_empty(&inode->i_dentry)) {
122 dentry = list_entry(inode->i_dentry.next,
123 struct dentry, d_alias);
124 if (dentry && dentry->d_name.name)
125 name = (const char *) dentry->d_name.name;
126 }
127
128 if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev"))
129 printk(KERN_DEBUG
130 "%s(%d): dirtied inode %lu (%s) on %s\n",
131 current->comm, task_pid_nr(current), inode->i_ino,
132 name, inode->i_sb->s_id);
133 }
134
135 spin_lock(&inode_lock);
136 if ((inode->i_state & flags) != flags) {
137 const int was_dirty = inode->i_state & I_DIRTY;
138
139 inode->i_state |= flags;
140
141
142
143
144
145
146 if (inode->i_state & I_SYNC)
147 goto out;
148
149
150
151
152
153 if (!S_ISBLK(inode->i_mode)) {
154 if (hlist_unhashed(&inode->i_hash))
155 goto out;
156 }
157 if (inode->i_state & (I_FREEING|I_CLEAR))
158 goto out;
159
160
161
162
163
164 if (!was_dirty) {
165 inode->dirtied_when = jiffies;
166 list_move(&inode->i_list, &sb->s_dirty);
167 }
168 }
169out:
170 spin_unlock(&inode_lock);
171}
172
173EXPORT_SYMBOL(__mark_inode_dirty);
174
175static int write_inode(struct inode *inode, int sync)
176{
177 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode))
178 return inode->i_sb->s_op->write_inode(inode, sync);
179 return 0;
180}
181
182
183
184
185
186
187
188
189
190
191static void redirty_tail(struct inode *inode)
192{
193 struct super_block *sb = inode->i_sb;
194
195 if (!list_empty(&sb->s_dirty)) {
196 struct inode *tail_inode;
197
198 tail_inode = list_entry(sb->s_dirty.next, struct inode, i_list);
199 if (!time_after_eq(inode->dirtied_when,
200 tail_inode->dirtied_when))
201 inode->dirtied_when = jiffies;
202 }
203 list_move(&inode->i_list, &sb->s_dirty);
204}
205
206
207
208
209static void requeue_io(struct inode *inode)
210{
211 list_move(&inode->i_list, &inode->i_sb->s_more_io);
212}
213
214static void inode_sync_complete(struct inode *inode)
215{
216
217
218
219 smp_mb();
220 wake_up_bit(&inode->i_state, __I_SYNC);
221}
222
223
224
225
226static void move_expired_inodes(struct list_head *delaying_queue,
227 struct list_head *dispatch_queue,
228 unsigned long *older_than_this)
229{
230 while (!list_empty(delaying_queue)) {
231 struct inode *inode = list_entry(delaying_queue->prev,
232 struct inode, i_list);
233 if (older_than_this &&
234 time_after(inode->dirtied_when, *older_than_this))
235 break;
236 list_move(&inode->i_list, dispatch_queue);
237 }
238}
239
240
241
242
243static void queue_io(struct super_block *sb,
244 unsigned long *older_than_this)
245{
246 list_splice_init(&sb->s_more_io, sb->s_io.prev);
247 move_expired_inodes(&sb->s_dirty, &sb->s_io, older_than_this);
248}
249
250int sb_has_dirty_inodes(struct super_block *sb)
251{
252 return !list_empty(&sb->s_dirty) ||
253 !list_empty(&sb->s_io) ||
254 !list_empty(&sb->s_more_io);
255}
256EXPORT_SYMBOL(sb_has_dirty_inodes);
257
258
259
260
261
262
263
264
265
266
267
268static int
269__sync_single_inode(struct inode *inode, struct writeback_control *wbc)
270{
271 unsigned dirty;
272 struct address_space *mapping = inode->i_mapping;
273 int wait = wbc->sync_mode == WB_SYNC_ALL;
274 int ret;
275
276 BUG_ON(inode->i_state & I_SYNC);
277 WARN_ON(inode->i_state & I_NEW);
278
279
280 dirty = inode->i_state & I_DIRTY;
281 inode->i_state |= I_SYNC;
282 inode->i_state &= ~I_DIRTY;
283
284 spin_unlock(&inode_lock);
285
286 ret = do_writepages(mapping, wbc);
287
288
289 if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
290 int err = write_inode(inode, wait);
291 if (ret == 0)
292 ret = err;
293 }
294
295 if (wait) {
296 int err = filemap_fdatawait(mapping);
297 if (ret == 0)
298 ret = err;
299 }
300
301 spin_lock(&inode_lock);
302 WARN_ON(inode->i_state & I_NEW);
303 inode->i_state &= ~I_SYNC;
304 if (!(inode->i_state & I_FREEING)) {
305 if (!(inode->i_state & I_DIRTY) &&
306 mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321 if (wbc->for_kupdate) {
322
323
324
325
326
327 inode->i_state |= I_DIRTY_PAGES;
328 if (wbc->nr_to_write <= 0) {
329
330
331
332 requeue_io(inode);
333 } else {
334
335
336
337 redirty_tail(inode);
338 }
339 } else {
340
341
342
343
344
345
346
347 inode->i_state |= I_DIRTY_PAGES;
348 redirty_tail(inode);
349 }
350 } else if (inode->i_state & I_DIRTY) {
351
352
353
354
355 redirty_tail(inode);
356 } else if (atomic_read(&inode->i_count)) {
357
358
359
360 list_move(&inode->i_list, &inode_in_use);
361 } else {
362
363
364
365 list_move(&inode->i_list, &inode_unused);
366 }
367 }
368 inode_sync_complete(inode);
369 return ret;
370}
371
372
373
374
375
376
377static int
378__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
379{
380 wait_queue_head_t *wqh;
381
382 if (!atomic_read(&inode->i_count))
383 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
384 else
385 WARN_ON(inode->i_state & I_WILL_FREE);
386
387 if ((wbc->sync_mode != WB_SYNC_ALL) && (inode->i_state & I_SYNC)) {
388
389
390
391
392
393
394
395 requeue_io(inode);
396 return 0;
397 }
398
399
400
401
402 if (inode->i_state & I_SYNC) {
403 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
404
405 wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
406 do {
407 spin_unlock(&inode_lock);
408 __wait_on_bit(wqh, &wq, inode_wait,
409 TASK_UNINTERRUPTIBLE);
410 spin_lock(&inode_lock);
411 } while (inode->i_state & I_SYNC);
412 }
413 return __sync_single_inode(inode, wbc);
414}
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441void generic_sync_sb_inodes(struct super_block *sb,
442 struct writeback_control *wbc)
443{
444 const unsigned long start = jiffies;
445 int sync = wbc->sync_mode == WB_SYNC_ALL;
446
447 spin_lock(&inode_lock);
448 if (!wbc->for_kupdate || list_empty(&sb->s_io))
449 queue_io(sb, wbc->older_than_this);
450
451 while (!list_empty(&sb->s_io)) {
452 struct inode *inode = list_entry(sb->s_io.prev,
453 struct inode, i_list);
454 struct address_space *mapping = inode->i_mapping;
455 struct backing_dev_info *bdi = mapping->backing_dev_info;
456 long pages_skipped;
457
458 if (!bdi_cap_writeback_dirty(bdi)) {
459 redirty_tail(inode);
460 if (sb_is_blkdev_sb(sb)) {
461
462
463
464
465 continue;
466 }
467
468
469
470
471
472 break;
473 }
474
475 if (inode->i_state & I_NEW) {
476 requeue_io(inode);
477 continue;
478 }
479
480 if (wbc->nonblocking && bdi_write_congested(bdi)) {
481 wbc->encountered_congestion = 1;
482 if (!sb_is_blkdev_sb(sb))
483 break;
484 requeue_io(inode);
485 continue;
486 }
487
488 if (wbc->bdi && bdi != wbc->bdi) {
489 if (!sb_is_blkdev_sb(sb))
490 break;
491 requeue_io(inode);
492 continue;
493 }
494
495
496 if (time_after(inode->dirtied_when, start))
497 break;
498
499
500 if (current_is_pdflush() && !writeback_acquire(bdi))
501 break;
502
503 BUG_ON(inode->i_state & I_FREEING);
504 __iget(inode);
505 pages_skipped = wbc->pages_skipped;
506 __writeback_single_inode(inode, wbc);
507 if (current_is_pdflush())
508 writeback_release(bdi);
509 if (wbc->pages_skipped != pages_skipped) {
510
511
512
513
514 redirty_tail(inode);
515 }
516 spin_unlock(&inode_lock);
517 iput(inode);
518 cond_resched();
519 spin_lock(&inode_lock);
520 if (wbc->nr_to_write <= 0) {
521 wbc->more_io = 1;
522 break;
523 }
524 if (!list_empty(&sb->s_more_io))
525 wbc->more_io = 1;
526 }
527
528 if (sync) {
529 struct inode *inode, *old_inode = NULL;
530
531
532
533
534
535
536
537
538 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
539 struct address_space *mapping;
540
541 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW))
542 continue;
543 mapping = inode->i_mapping;
544 if (mapping->nrpages == 0)
545 continue;
546 __iget(inode);
547 spin_unlock(&inode_lock);
548
549
550
551
552
553
554
555
556 iput(old_inode);
557 old_inode = inode;
558
559 filemap_fdatawait(mapping);
560
561 cond_resched();
562
563 spin_lock(&inode_lock);
564 }
565 spin_unlock(&inode_lock);
566 iput(old_inode);
567 } else
568 spin_unlock(&inode_lock);
569
570 return;
571}
572EXPORT_SYMBOL_GPL(generic_sync_sb_inodes);
573
574static void sync_sb_inodes(struct super_block *sb,
575 struct writeback_control *wbc)
576{
577 generic_sync_sb_inodes(sb, wbc);
578}
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599void
600writeback_inodes(struct writeback_control *wbc)
601{
602 struct super_block *sb;
603
604 might_sleep();
605 spin_lock(&sb_lock);
606restart:
607 list_for_each_entry_reverse(sb, &super_blocks, s_list) {
608 if (sb_has_dirty_inodes(sb)) {
609
610 sb->s_count++;
611 spin_unlock(&sb_lock);
612
613
614
615
616
617 if (down_read_trylock(&sb->s_umount)) {
618 if (sb->s_root)
619 sync_sb_inodes(sb, wbc);
620 up_read(&sb->s_umount);
621 }
622 spin_lock(&sb_lock);
623 if (__put_super_and_need_restart(sb))
624 goto restart;
625 }
626 if (wbc->nr_to_write <= 0)
627 break;
628 }
629 spin_unlock(&sb_lock);
630}
631
632
633
634
635
636
637
638
639
640
641
642void sync_inodes_sb(struct super_block *sb, int wait)
643{
644 struct writeback_control wbc = {
645 .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
646 .range_start = 0,
647 .range_end = LLONG_MAX,
648 };
649
650 if (!wait) {
651 unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY);
652 unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS);
653
654 wbc.nr_to_write = nr_dirty + nr_unstable +
655 (inodes_stat.nr_inodes - inodes_stat.nr_unused);
656 } else
657 wbc.nr_to_write = LONG_MAX;
658
659 sync_sb_inodes(sb, &wbc);
660}
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681static void __sync_inodes(int wait)
682{
683 struct super_block *sb;
684
685 spin_lock(&sb_lock);
686restart:
687 list_for_each_entry(sb, &super_blocks, s_list) {
688 sb->s_count++;
689 spin_unlock(&sb_lock);
690 down_read(&sb->s_umount);
691 if (sb->s_root) {
692 sync_inodes_sb(sb, wait);
693 sync_blockdev(sb->s_bdev);
694 }
695 up_read(&sb->s_umount);
696 spin_lock(&sb_lock);
697 if (__put_super_and_need_restart(sb))
698 goto restart;
699 }
700 spin_unlock(&sb_lock);
701}
702
703void sync_inodes(int wait)
704{
705 __sync_inodes(0);
706
707 if (wait)
708 __sync_inodes(1);
709}
710
711
712
713
714
715
716
717
718
719
720
721int write_inode_now(struct inode *inode, int sync)
722{
723 int ret;
724 struct writeback_control wbc = {
725 .nr_to_write = LONG_MAX,
726 .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
727 .range_start = 0,
728 .range_end = LLONG_MAX,
729 };
730
731 if (!mapping_cap_writeback_dirty(inode->i_mapping))
732 wbc.nr_to_write = 0;
733
734 might_sleep();
735 spin_lock(&inode_lock);
736 ret = __writeback_single_inode(inode, &wbc);
737 spin_unlock(&inode_lock);
738 if (sync)
739 inode_sync_wait(inode);
740 return ret;
741}
742EXPORT_SYMBOL(write_inode_now);
743
744
745
746
747
748
749
750
751
752
753
754
755int sync_inode(struct inode *inode, struct writeback_control *wbc)
756{
757 int ret;
758
759 spin_lock(&inode_lock);
760 ret = __writeback_single_inode(inode, wbc);
761 spin_unlock(&inode_lock);
762 return ret;
763}
764EXPORT_SYMBOL(sync_inode);
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783int generic_osync_inode(struct inode *inode, struct address_space *mapping, int what)
784{
785 int err = 0;
786 int need_write_inode_now = 0;
787 int err2;
788
789 if (what & OSYNC_DATA)
790 err = filemap_fdatawrite(mapping);
791 if (what & (OSYNC_METADATA|OSYNC_DATA)) {
792 err2 = sync_mapping_buffers(mapping);
793 if (!err)
794 err = err2;
795 }
796 if (what & OSYNC_DATA) {
797 err2 = filemap_fdatawait(mapping);
798 if (!err)
799 err = err2;
800 }
801
802 spin_lock(&inode_lock);
803 if ((inode->i_state & I_DIRTY) &&
804 ((what & OSYNC_INODE) || (inode->i_state & I_DIRTY_DATASYNC)))
805 need_write_inode_now = 1;
806 spin_unlock(&inode_lock);
807
808 if (need_write_inode_now) {
809 err2 = write_inode_now(inode, 1);
810 if (!err)
811 err = err2;
812 }
813 else
814 inode_sync_wait(inode);
815
816 return err;
817}
818EXPORT_SYMBOL(generic_osync_inode);
819