linux/fs/fs-writeback.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * fs/fs-writeback.c
   4 *
   5 * Copyright (C) 2002, Linus Torvalds.
   6 *
   7 * Contains all the functions related to writing back and waiting
   8 * upon dirty inodes against superblocks, and writing back dirty
   9 * pages against inodes.  ie: data writeback.  Writeout of the
  10 * inode itself is not handled here.
  11 *
  12 * 10Apr2002    Andrew Morton
  13 *              Split out of fs/inode.c
  14 *              Additions for address_space-based writeback
  15 */
  16
  17#include <linux/kernel.h>
  18#include <linux/export.h>
  19#include <linux/spinlock.h>
  20#include <linux/slab.h>
  21#include <linux/sched.h>
  22#include <linux/fs.h>
  23#include <linux/mm.h>
  24#include <linux/pagemap.h>
  25#include <linux/kthread.h>
  26#include <linux/writeback.h>
  27#include <linux/blkdev.h>
  28#include <linux/backing-dev.h>
  29#include <linux/tracepoint.h>
  30#include <linux/device.h>
  31#include <linux/memcontrol.h>
  32#include "internal.h"
  33
  34/*
  35 * 4MB minimal write chunk size
  36 */
  37#define MIN_WRITEBACK_PAGES     (4096UL >> (PAGE_SHIFT - 10))
  38
  39/*
  40 * Passed into wb_writeback(), essentially a subset of writeback_control
  41 */
  42struct wb_writeback_work {
  43        long nr_pages;
  44        struct super_block *sb;
  45        enum writeback_sync_modes sync_mode;
  46        unsigned int tagged_writepages:1;
  47        unsigned int for_kupdate:1;
  48        unsigned int range_cyclic:1;
  49        unsigned int for_background:1;
  50        unsigned int for_sync:1;        /* sync(2) WB_SYNC_ALL writeback */
  51        unsigned int auto_free:1;       /* free on completion */
  52        enum wb_reason reason;          /* why was writeback initiated? */
  53
  54        struct list_head list;          /* pending work list */
  55        struct wb_completion *done;     /* set if the caller waits */
  56};
  57
  58/*
  59 * If an inode is constantly having its pages dirtied, but then the
  60 * updates stop dirtytime_expire_interval seconds in the past, it's
  61 * possible for the worst case time between when an inode has its
  62 * timestamps updated and when they finally get written out to be two
  63 * dirtytime_expire_intervals.  We set the default to 12 hours (in
  64 * seconds), which means most of the time inodes will have their
  65 * timestamps written to disk after 12 hours, but in the worst case a
  66 * few inodes might not their timestamps updated for 24 hours.
  67 */
  68unsigned int dirtytime_expire_interval = 12 * 60 * 60;
  69
  70static inline struct inode *wb_inode(struct list_head *head)
  71{
  72        return list_entry(head, struct inode, i_io_list);
  73}
  74
  75/*
  76 * Include the creation of the trace points after defining the
  77 * wb_writeback_work structure and inline functions so that the definition
  78 * remains local to this file.
  79 */
  80#define CREATE_TRACE_POINTS
  81#include <trace/events/writeback.h>
  82
  83EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage);
  84
  85static bool wb_io_lists_populated(struct bdi_writeback *wb)
  86{
  87        if (wb_has_dirty_io(wb)) {
  88                return false;
  89        } else {
  90                set_bit(WB_has_dirty_io, &wb->state);
  91                WARN_ON_ONCE(!wb->avg_write_bandwidth);
  92                atomic_long_add(wb->avg_write_bandwidth,
  93                                &wb->bdi->tot_write_bandwidth);
  94                return true;
  95        }
  96}
  97
  98static void wb_io_lists_depopulated(struct bdi_writeback *wb)
  99{
 100        if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&
 101            list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {
 102                clear_bit(WB_has_dirty_io, &wb->state);
 103                WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,
 104                                        &wb->bdi->tot_write_bandwidth) < 0);
 105        }
 106}
 107
 108/**
 109 * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list
 110 * @inode: inode to be moved
 111 * @wb: target bdi_writeback
 112 * @head: one of @wb->b_{dirty|io|more_io|dirty_time}
 113 *
 114 * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io.
 115 * Returns %true if @inode is the first occupant of the !dirty_time IO
 116 * lists; otherwise, %false.
 117 */
 118static bool inode_io_list_move_locked(struct inode *inode,
 119                                      struct bdi_writeback *wb,
 120                                      struct list_head *head)
 121{
 122        assert_spin_locked(&wb->list_lock);
 123
 124        list_move(&inode->i_io_list, head);
 125
 126        /* dirty_time doesn't count as dirty_io until expiration */
 127        if (head != &wb->b_dirty_time)
 128                return wb_io_lists_populated(wb);
 129
 130        wb_io_lists_depopulated(wb);
 131        return false;
 132}
 133
 134/**
 135 * inode_io_list_del_locked - remove an inode from its bdi_writeback IO list
 136 * @inode: inode to be removed
 137 * @wb: bdi_writeback @inode is being removed from
 138 *
 139 * Remove @inode which may be on one of @wb->b_{dirty|io|more_io} lists and
 140 * clear %WB_has_dirty_io if all are empty afterwards.
 141 */
 142static void inode_io_list_del_locked(struct inode *inode,
 143                                     struct bdi_writeback *wb)
 144{
 145        assert_spin_locked(&wb->list_lock);
 146        assert_spin_locked(&inode->i_lock);
 147
 148        inode->i_state &= ~I_SYNC_QUEUED;
 149        list_del_init(&inode->i_io_list);
 150        wb_io_lists_depopulated(wb);
 151}
 152
 153static void wb_wakeup(struct bdi_writeback *wb)
 154{
 155        spin_lock_bh(&wb->work_lock);
 156        if (test_bit(WB_registered, &wb->state))
 157                mod_delayed_work(bdi_wq, &wb->dwork, 0);
 158        spin_unlock_bh(&wb->work_lock);
 159}
 160
 161static void finish_writeback_work(struct bdi_writeback *wb,
 162                                  struct wb_writeback_work *work)
 163{
 164        struct wb_completion *done = work->done;
 165
 166        if (work->auto_free)
 167                kfree(work);
 168        if (done) {
 169                wait_queue_head_t *waitq = done->waitq;
 170
 171                /* @done can't be accessed after the following dec */
 172                if (atomic_dec_and_test(&done->cnt))
 173                        wake_up_all(waitq);
 174        }
 175}
 176
 177static void wb_queue_work(struct bdi_writeback *wb,
 178                          struct wb_writeback_work *work)
 179{
 180        trace_writeback_queue(wb, work);
 181
 182        if (work->done)
 183                atomic_inc(&work->done->cnt);
 184
 185        spin_lock_bh(&wb->work_lock);
 186
 187        if (test_bit(WB_registered, &wb->state)) {
 188                list_add_tail(&work->list, &wb->work_list);
 189                mod_delayed_work(bdi_wq, &wb->dwork, 0);
 190        } else
 191                finish_writeback_work(wb, work);
 192
 193        spin_unlock_bh(&wb->work_lock);
 194}
 195
 196/**
 197 * wb_wait_for_completion - wait for completion of bdi_writeback_works
 198 * @done: target wb_completion
 199 *
 200 * Wait for one or more work items issued to @bdi with their ->done field
 201 * set to @done, which should have been initialized with
 202 * DEFINE_WB_COMPLETION().  This function returns after all such work items
 203 * are completed.  Work items which are waited upon aren't freed
 204 * automatically on completion.
 205 */
 206void wb_wait_for_completion(struct wb_completion *done)
 207{
 208        atomic_dec(&done->cnt);         /* put down the initial count */
 209        wait_event(*done->waitq, !atomic_read(&done->cnt));
 210}
 211
 212#ifdef CONFIG_CGROUP_WRITEBACK
 213
 214/*
 215 * Parameters for foreign inode detection, see wbc_detach_inode() to see
 216 * how they're used.
 217 *
 218 * These paramters are inherently heuristical as the detection target
 219 * itself is fuzzy.  All we want to do is detaching an inode from the
 220 * current owner if it's being written to by some other cgroups too much.
 221 *
 222 * The current cgroup writeback is built on the assumption that multiple
 223 * cgroups writing to the same inode concurrently is very rare and a mode
 224 * of operation which isn't well supported.  As such, the goal is not
 225 * taking too long when a different cgroup takes over an inode while
 226 * avoiding too aggressive flip-flops from occasional foreign writes.
 227 *
 228 * We record, very roughly, 2s worth of IO time history and if more than
 229 * half of that is foreign, trigger the switch.  The recording is quantized
 230 * to 16 slots.  To avoid tiny writes from swinging the decision too much,
 231 * writes smaller than 1/8 of avg size are ignored.
 232 */
 233#define WB_FRN_TIME_SHIFT       13      /* 1s = 2^13, upto 8 secs w/ 16bit */
 234#define WB_FRN_TIME_AVG_SHIFT   3       /* avg = avg * 7/8 + new * 1/8 */
 235#define WB_FRN_TIME_CUT_DIV     8       /* ignore rounds < avg / 8 */
 236#define WB_FRN_TIME_PERIOD      (2 * (1 << WB_FRN_TIME_SHIFT))  /* 2s */
 237
 238#define WB_FRN_HIST_SLOTS       16      /* inode->i_wb_frn_history is 16bit */
 239#define WB_FRN_HIST_UNIT        (WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS)
 240                                        /* each slot's duration is 2s / 16 */
 241#define WB_FRN_HIST_THR_SLOTS   (WB_FRN_HIST_SLOTS / 2)
 242                                        /* if foreign slots >= 8, switch */
 243#define WB_FRN_HIST_MAX_SLOTS   (WB_FRN_HIST_THR_SLOTS / 2 + 1)
 244                                        /* one round can affect upto 5 slots */
 245#define WB_FRN_MAX_IN_FLIGHT    1024    /* don't queue too many concurrently */
 246
 247static atomic_t isw_nr_in_flight = ATOMIC_INIT(0);
 248static struct workqueue_struct *isw_wq;
 249
 250void __inode_attach_wb(struct inode *inode, struct page *page)
 251{
 252        struct backing_dev_info *bdi = inode_to_bdi(inode);
 253        struct bdi_writeback *wb = NULL;
 254
 255        if (inode_cgwb_enabled(inode)) {
 256                struct cgroup_subsys_state *memcg_css;
 257
 258                if (page) {
 259                        memcg_css = mem_cgroup_css_from_page(page);
 260                        wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
 261                } else {
 262                        /* must pin memcg_css, see wb_get_create() */
 263                        memcg_css = task_get_css(current, memory_cgrp_id);
 264                        wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
 265                        css_put(memcg_css);
 266                }
 267        }
 268
 269        if (!wb)
 270                wb = &bdi->wb;
 271
 272        /*
 273         * There may be multiple instances of this function racing to
 274         * update the same inode.  Use cmpxchg() to tell the winner.
 275         */
 276        if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))
 277                wb_put(wb);
 278}
 279EXPORT_SYMBOL_GPL(__inode_attach_wb);
 280
 281/**
 282 * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it
 283 * @inode: inode of interest with i_lock held
 284 *
 285 * Returns @inode's wb with its list_lock held.  @inode->i_lock must be
 286 * held on entry and is released on return.  The returned wb is guaranteed
 287 * to stay @inode's associated wb until its list_lock is released.
 288 */
 289static struct bdi_writeback *
 290locked_inode_to_wb_and_lock_list(struct inode *inode)
 291        __releases(&inode->i_lock)
 292        __acquires(&wb->list_lock)
 293{
 294        while (true) {
 295                struct bdi_writeback *wb = inode_to_wb(inode);
 296
 297                /*
 298                 * inode_to_wb() association is protected by both
 299                 * @inode->i_lock and @wb->list_lock but list_lock nests
 300                 * outside i_lock.  Drop i_lock and verify that the
 301                 * association hasn't changed after acquiring list_lock.
 302                 */
 303                wb_get(wb);
 304                spin_unlock(&inode->i_lock);
 305                spin_lock(&wb->list_lock);
 306
 307                /* i_wb may have changed inbetween, can't use inode_to_wb() */
 308                if (likely(wb == inode->i_wb)) {
 309                        wb_put(wb);     /* @inode already has ref */
 310                        return wb;
 311                }
 312
 313                spin_unlock(&wb->list_lock);
 314                wb_put(wb);
 315                cpu_relax();
 316                spin_lock(&inode->i_lock);
 317        }
 318}
 319
 320/**
 321 * inode_to_wb_and_lock_list - determine an inode's wb and lock it
 322 * @inode: inode of interest
 323 *
 324 * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held
 325 * on entry.
 326 */
 327static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
 328        __acquires(&wb->list_lock)
 329{
 330        spin_lock(&inode->i_lock);
 331        return locked_inode_to_wb_and_lock_list(inode);
 332}
 333
 334struct inode_switch_wbs_context {
 335        struct inode            *inode;
 336        struct bdi_writeback    *new_wb;
 337
 338        struct rcu_head         rcu_head;
 339        struct work_struct      work;
 340};
 341
 342static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi)
 343{
 344        down_write(&bdi->wb_switch_rwsem);
 345}
 346
 347static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi)
 348{
 349        up_write(&bdi->wb_switch_rwsem);
 350}
 351
 352static void inode_switch_wbs_work_fn(struct work_struct *work)
 353{
 354        struct inode_switch_wbs_context *isw =
 355                container_of(work, struct inode_switch_wbs_context, work);
 356        struct inode *inode = isw->inode;
 357        struct backing_dev_info *bdi = inode_to_bdi(inode);
 358        struct address_space *mapping = inode->i_mapping;
 359        struct bdi_writeback *old_wb = inode->i_wb;
 360        struct bdi_writeback *new_wb = isw->new_wb;
 361        XA_STATE(xas, &mapping->i_pages, 0);
 362        struct page *page;
 363        bool switched = false;
 364
 365        /*
 366         * If @inode switches cgwb membership while sync_inodes_sb() is
 367         * being issued, sync_inodes_sb() might miss it.  Synchronize.
 368         */
 369        down_read(&bdi->wb_switch_rwsem);
 370
 371        /*
 372         * By the time control reaches here, RCU grace period has passed
 373         * since I_WB_SWITCH assertion and all wb stat update transactions
 374         * between unlocked_inode_to_wb_begin/end() are guaranteed to be
 375         * synchronizing against the i_pages lock.
 376         *
 377         * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock
 378         * gives us exclusion against all wb related operations on @inode
 379         * including IO list manipulations and stat updates.
 380         */
 381        if (old_wb < new_wb) {
 382                spin_lock(&old_wb->list_lock);
 383                spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING);
 384        } else {
 385                spin_lock(&new_wb->list_lock);
 386                spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING);
 387        }
 388        spin_lock(&inode->i_lock);
 389        xa_lock_irq(&mapping->i_pages);
 390
 391        /*
 392         * Once I_FREEING is visible under i_lock, the eviction path owns
 393         * the inode and we shouldn't modify ->i_io_list.
 394         */
 395        if (unlikely(inode->i_state & I_FREEING))
 396                goto skip_switch;
 397
 398        trace_inode_switch_wbs(inode, old_wb, new_wb);
 399
 400        /*
 401         * Count and transfer stats.  Note that PAGECACHE_TAG_DIRTY points
 402         * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to
 403         * pages actually under writeback.
 404         */
 405        xas_for_each_marked(&xas, page, ULONG_MAX, PAGECACHE_TAG_DIRTY) {
 406                if (PageDirty(page)) {
 407                        dec_wb_stat(old_wb, WB_RECLAIMABLE);
 408                        inc_wb_stat(new_wb, WB_RECLAIMABLE);
 409                }
 410        }
 411
 412        xas_set(&xas, 0);
 413        xas_for_each_marked(&xas, page, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) {
 414                WARN_ON_ONCE(!PageWriteback(page));
 415                dec_wb_stat(old_wb, WB_WRITEBACK);
 416                inc_wb_stat(new_wb, WB_WRITEBACK);
 417        }
 418
 419        wb_get(new_wb);
 420
 421        /*
 422         * Transfer to @new_wb's IO list if necessary.  The specific list
 423         * @inode was on is ignored and the inode is put on ->b_dirty which
 424         * is always correct including from ->b_dirty_time.  The transfer
 425         * preserves @inode->dirtied_when ordering.
 426         */
 427        if (!list_empty(&inode->i_io_list)) {
 428                struct inode *pos;
 429
 430                inode_io_list_del_locked(inode, old_wb);
 431                inode->i_wb = new_wb;
 432                list_for_each_entry(pos, &new_wb->b_dirty, i_io_list)
 433                        if (time_after_eq(inode->dirtied_when,
 434                                          pos->dirtied_when))
 435                                break;
 436                inode_io_list_move_locked(inode, new_wb, pos->i_io_list.prev);
 437        } else {
 438                inode->i_wb = new_wb;
 439        }
 440
 441        /* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */
 442        inode->i_wb_frn_winner = 0;
 443        inode->i_wb_frn_avg_time = 0;
 444        inode->i_wb_frn_history = 0;
 445        switched = true;
 446skip_switch:
 447        /*
 448         * Paired with load_acquire in unlocked_inode_to_wb_begin() and
 449         * ensures that the new wb is visible if they see !I_WB_SWITCH.
 450         */
 451        smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH);
 452
 453        xa_unlock_irq(&mapping->i_pages);
 454        spin_unlock(&inode->i_lock);
 455        spin_unlock(&new_wb->list_lock);
 456        spin_unlock(&old_wb->list_lock);
 457
 458        up_read(&bdi->wb_switch_rwsem);
 459
 460        if (switched) {
 461                wb_wakeup(new_wb);
 462                wb_put(old_wb);
 463        }
 464        wb_put(new_wb);
 465
 466        iput(inode);
 467        kfree(isw);
 468
 469        atomic_dec(&isw_nr_in_flight);
 470}
 471
 472static void inode_switch_wbs_rcu_fn(struct rcu_head *rcu_head)
 473{
 474        struct inode_switch_wbs_context *isw = container_of(rcu_head,
 475                                struct inode_switch_wbs_context, rcu_head);
 476
 477        /* needs to grab bh-unsafe locks, bounce to work item */
 478        INIT_WORK(&isw->work, inode_switch_wbs_work_fn);
 479        queue_work(isw_wq, &isw->work);
 480}
 481
 482/**
 483 * inode_switch_wbs - change the wb association of an inode
 484 * @inode: target inode
 485 * @new_wb_id: ID of the new wb
 486 *
 487 * Switch @inode's wb association to the wb identified by @new_wb_id.  The
 488 * switching is performed asynchronously and may fail silently.
 489 */
 490static void inode_switch_wbs(struct inode *inode, int new_wb_id)
 491{
 492        struct backing_dev_info *bdi = inode_to_bdi(inode);
 493        struct cgroup_subsys_state *memcg_css;
 494        struct inode_switch_wbs_context *isw;
 495
 496        /* noop if seems to be already in progress */
 497        if (inode->i_state & I_WB_SWITCH)
 498                return;
 499
 500        /* avoid queueing a new switch if too many are already in flight */
 501        if (atomic_read(&isw_nr_in_flight) > WB_FRN_MAX_IN_FLIGHT)
 502                return;
 503
 504        isw = kzalloc(sizeof(*isw), GFP_ATOMIC);
 505        if (!isw)
 506                return;
 507
 508        atomic_inc(&isw_nr_in_flight);
 509
 510        /* find and pin the new wb */
 511        rcu_read_lock();
 512        memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys);
 513        if (memcg_css && !css_tryget(memcg_css))
 514                memcg_css = NULL;
 515        rcu_read_unlock();
 516        if (!memcg_css)
 517                goto out_free;
 518
 519        isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
 520        css_put(memcg_css);
 521        if (!isw->new_wb)
 522                goto out_free;
 523
 524        /* while holding I_WB_SWITCH, no one else can update the association */
 525        spin_lock(&inode->i_lock);
 526        if (!(inode->i_sb->s_flags & SB_ACTIVE) ||
 527            inode->i_state & (I_WB_SWITCH | I_FREEING) ||
 528            inode_to_wb(inode) == isw->new_wb) {
 529                spin_unlock(&inode->i_lock);
 530                goto out_free;
 531        }
 532        inode->i_state |= I_WB_SWITCH;
 533        __iget(inode);
 534        spin_unlock(&inode->i_lock);
 535
 536        isw->inode = inode;
 537
 538        /*
 539         * In addition to synchronizing among switchers, I_WB_SWITCH tells
 540         * the RCU protected stat update paths to grab the i_page
 541         * lock so that stat transfer can synchronize against them.
 542         * Let's continue after I_WB_SWITCH is guaranteed to be visible.
 543         */
 544        call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn);
 545        return;
 546
 547out_free:
 548        atomic_dec(&isw_nr_in_flight);
 549        if (isw->new_wb)
 550                wb_put(isw->new_wb);
 551        kfree(isw);
 552}
 553
 554/**
 555 * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it
 556 * @wbc: writeback_control of interest
 557 * @inode: target inode
 558 *
 559 * @inode is locked and about to be written back under the control of @wbc.
 560 * Record @inode's writeback context into @wbc and unlock the i_lock.  On
 561 * writeback completion, wbc_detach_inode() should be called.  This is used
 562 * to track the cgroup writeback context.
 563 */
 564void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
 565                                 struct inode *inode)
 566{
 567        if (!inode_cgwb_enabled(inode)) {
 568                spin_unlock(&inode->i_lock);
 569                return;
 570        }
 571
 572        wbc->wb = inode_to_wb(inode);
 573        wbc->inode = inode;
 574
 575        wbc->wb_id = wbc->wb->memcg_css->id;
 576        wbc->wb_lcand_id = inode->i_wb_frn_winner;
 577        wbc->wb_tcand_id = 0;
 578        wbc->wb_bytes = 0;
 579        wbc->wb_lcand_bytes = 0;
 580        wbc->wb_tcand_bytes = 0;
 581
 582        wb_get(wbc->wb);
 583        spin_unlock(&inode->i_lock);
 584
 585        /*
 586         * A dying wb indicates that either the blkcg associated with the
 587         * memcg changed or the associated memcg is dying.  In the first
 588         * case, a replacement wb should already be available and we should
 589         * refresh the wb immediately.  In the second case, trying to
 590         * refresh will keep failing.
 591         */
 592        if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css)))
 593                inode_switch_wbs(inode, wbc->wb_id);
 594}
 595EXPORT_SYMBOL_GPL(wbc_attach_and_unlock_inode);
 596
 597/**
 598 * wbc_detach_inode - disassociate wbc from inode and perform foreign detection
 599 * @wbc: writeback_control of the just finished writeback
 600 *
 601 * To be called after a writeback attempt of an inode finishes and undoes
 602 * wbc_attach_and_unlock_inode().  Can be called under any context.
 603 *
 604 * As concurrent write sharing of an inode is expected to be very rare and
 605 * memcg only tracks page ownership on first-use basis severely confining
 606 * the usefulness of such sharing, cgroup writeback tracks ownership
 607 * per-inode.  While the support for concurrent write sharing of an inode
 608 * is deemed unnecessary, an inode being written to by different cgroups at
 609 * different points in time is a lot more common, and, more importantly,
 610 * charging only by first-use can too readily lead to grossly incorrect
 611 * behaviors (single foreign page can lead to gigabytes of writeback to be
 612 * incorrectly attributed).
 613 *
 614 * To resolve this issue, cgroup writeback detects the majority dirtier of
 615 * an inode and transfers the ownership to it.  To avoid unnnecessary
 616 * oscillation, the detection mechanism keeps track of history and gives
 617 * out the switch verdict only if the foreign usage pattern is stable over
 618 * a certain amount of time and/or writeback attempts.
 619 *
 620 * On each writeback attempt, @wbc tries to detect the majority writer
 621 * using Boyer-Moore majority vote algorithm.  In addition to the byte
 622 * count from the majority voting, it also counts the bytes written for the
 623 * current wb and the last round's winner wb (max of last round's current
 624 * wb, the winner from two rounds ago, and the last round's majority
 625 * candidate).  Keeping track of the historical winner helps the algorithm
 626 * to semi-reliably detect the most active writer even when it's not the
 627 * absolute majority.
 628 *
 629 * Once the winner of the round is determined, whether the winner is
 630 * foreign or not and how much IO time the round consumed is recorded in
 631 * inode->i_wb_frn_history.  If the amount of recorded foreign IO time is
 632 * over a certain threshold, the switch verdict is given.
 633 */
 634void wbc_detach_inode(struct writeback_control *wbc)
 635{
 636        struct bdi_writeback *wb = wbc->wb;
 637        struct inode *inode = wbc->inode;
 638        unsigned long avg_time, max_bytes, max_time;
 639        u16 history;
 640        int max_id;
 641
 642        if (!wb)
 643                return;
 644
 645        history = inode->i_wb_frn_history;
 646        avg_time = inode->i_wb_frn_avg_time;
 647
 648        /* pick the winner of this round */
 649        if (wbc->wb_bytes >= wbc->wb_lcand_bytes &&
 650            wbc->wb_bytes >= wbc->wb_tcand_bytes) {
 651                max_id = wbc->wb_id;
 652                max_bytes = wbc->wb_bytes;
 653        } else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) {
 654                max_id = wbc->wb_lcand_id;
 655                max_bytes = wbc->wb_lcand_bytes;
 656        } else {
 657                max_id = wbc->wb_tcand_id;
 658                max_bytes = wbc->wb_tcand_bytes;
 659        }
 660
 661        /*
 662         * Calculate the amount of IO time the winner consumed and fold it
 663         * into the running average kept per inode.  If the consumed IO
 664         * time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for
 665         * deciding whether to switch or not.  This is to prevent one-off
 666         * small dirtiers from skewing the verdict.
 667         */
 668        max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT,
 669                                wb->avg_write_bandwidth);
 670        if (avg_time)
 671                avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) -
 672                            (avg_time >> WB_FRN_TIME_AVG_SHIFT);
 673        else
 674                avg_time = max_time;    /* immediate catch up on first run */
 675
 676        if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) {
 677                int slots;
 678
 679                /*
 680                 * The switch verdict is reached if foreign wb's consume
 681                 * more than a certain proportion of IO time in a
 682                 * WB_FRN_TIME_PERIOD.  This is loosely tracked by 16 slot
 683                 * history mask where each bit represents one sixteenth of
 684                 * the period.  Determine the number of slots to shift into
 685                 * history from @max_time.
 686                 */
 687                slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT),
 688                            (unsigned long)WB_FRN_HIST_MAX_SLOTS);
 689                history <<= slots;
 690                if (wbc->wb_id != max_id)
 691                        history |= (1U << slots) - 1;
 692
 693                if (history)
 694                        trace_inode_foreign_history(inode, wbc, history);
 695
 696                /*
 697                 * Switch if the current wb isn't the consistent winner.
 698                 * If there are multiple closely competing dirtiers, the
 699                 * inode may switch across them repeatedly over time, which
 700                 * is okay.  The main goal is avoiding keeping an inode on
 701                 * the wrong wb for an extended period of time.
 702                 */
 703                if (hweight32(history) > WB_FRN_HIST_THR_SLOTS)
 704                        inode_switch_wbs(inode, max_id);
 705        }
 706
 707        /*
 708         * Multiple instances of this function may race to update the
 709         * following fields but we don't mind occassional inaccuracies.
 710         */
 711        inode->i_wb_frn_winner = max_id;
 712        inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX);
 713        inode->i_wb_frn_history = history;
 714
 715        wb_put(wbc->wb);
 716        wbc->wb = NULL;
 717}
 718EXPORT_SYMBOL_GPL(wbc_detach_inode);
 719
 720/**
 721 * wbc_account_cgroup_owner - account writeback to update inode cgroup ownership
 722 * @wbc: writeback_control of the writeback in progress
 723 * @page: page being written out
 724 * @bytes: number of bytes being written out
 725 *
 726 * @bytes from @page are about to written out during the writeback
 727 * controlled by @wbc.  Keep the book for foreign inode detection.  See
 728 * wbc_detach_inode().
 729 */
 730void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page,
 731                              size_t bytes)
 732{
 733        struct cgroup_subsys_state *css;
 734        int id;
 735
 736        /*
 737         * pageout() path doesn't attach @wbc to the inode being written
 738         * out.  This is intentional as we don't want the function to block
 739         * behind a slow cgroup.  Ultimately, we want pageout() to kick off
 740         * regular writeback instead of writing things out itself.
 741         */
 742        if (!wbc->wb || wbc->no_cgroup_owner)
 743                return;
 744
 745        css = mem_cgroup_css_from_page(page);
 746        /* dead cgroups shouldn't contribute to inode ownership arbitration */
 747        if (!(css->flags & CSS_ONLINE))
 748                return;
 749
 750        id = css->id;
 751
 752        if (id == wbc->wb_id) {
 753                wbc->wb_bytes += bytes;
 754                return;
 755        }
 756
 757        if (id == wbc->wb_lcand_id)
 758                wbc->wb_lcand_bytes += bytes;
 759
 760        /* Boyer-Moore majority vote algorithm */
 761        if (!wbc->wb_tcand_bytes)
 762                wbc->wb_tcand_id = id;
 763        if (id == wbc->wb_tcand_id)
 764                wbc->wb_tcand_bytes += bytes;
 765        else
 766                wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes);
 767}
 768EXPORT_SYMBOL_GPL(wbc_account_cgroup_owner);
 769
 770/**
 771 * inode_congested - test whether an inode is congested
 772 * @inode: inode to test for congestion (may be NULL)
 773 * @cong_bits: mask of WB_[a]sync_congested bits to test
 774 *
 775 * Tests whether @inode is congested.  @cong_bits is the mask of congestion
 776 * bits to test and the return value is the mask of set bits.
 777 *
 778 * If cgroup writeback is enabled for @inode, the congestion state is
 779 * determined by whether the cgwb (cgroup bdi_writeback) for the blkcg
 780 * associated with @inode is congested; otherwise, the root wb's congestion
 781 * state is used.
 782 *
 783 * @inode is allowed to be NULL as this function is often called on
 784 * mapping->host which is NULL for the swapper space.
 785 */
 786int inode_congested(struct inode *inode, int cong_bits)
 787{
 788        /*
 789         * Once set, ->i_wb never becomes NULL while the inode is alive.
 790         * Start transaction iff ->i_wb is visible.
 791         */
 792        if (inode && inode_to_wb_is_valid(inode)) {
 793                struct bdi_writeback *wb;
 794                struct wb_lock_cookie lock_cookie = {};
 795                bool congested;
 796
 797                wb = unlocked_inode_to_wb_begin(inode, &lock_cookie);
 798                congested = wb_congested(wb, cong_bits);
 799                unlocked_inode_to_wb_end(inode, &lock_cookie);
 800                return congested;
 801        }
 802
 803        return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
 804}
 805EXPORT_SYMBOL_GPL(inode_congested);
 806
 807/**
 808 * wb_split_bdi_pages - split nr_pages to write according to bandwidth
 809 * @wb: target bdi_writeback to split @nr_pages to
 810 * @nr_pages: number of pages to write for the whole bdi
 811 *
 812 * Split @wb's portion of @nr_pages according to @wb's write bandwidth in
 813 * relation to the total write bandwidth of all wb's w/ dirty inodes on
 814 * @wb->bdi.
 815 */
 816static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
 817{
 818        unsigned long this_bw = wb->avg_write_bandwidth;
 819        unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
 820
 821        if (nr_pages == LONG_MAX)
 822                return LONG_MAX;
 823
 824        /*
 825         * This may be called on clean wb's and proportional distribution
 826         * may not make sense, just use the original @nr_pages in those
 827         * cases.  In general, we wanna err on the side of writing more.
 828         */
 829        if (!tot_bw || this_bw >= tot_bw)
 830                return nr_pages;
 831        else
 832                return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw);
 833}
 834
 835/**
 836 * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi
 837 * @bdi: target backing_dev_info
 838 * @base_work: wb_writeback_work to issue
 839 * @skip_if_busy: skip wb's which already have writeback in progress
 840 *
 841 * Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which
 842 * have dirty inodes.  If @base_work->nr_page isn't %LONG_MAX, it's
 843 * distributed to the busy wbs according to each wb's proportion in the
 844 * total active write bandwidth of @bdi.
 845 */
 846static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
 847                                  struct wb_writeback_work *base_work,
 848                                  bool skip_if_busy)
 849{
 850        struct bdi_writeback *last_wb = NULL;
 851        struct bdi_writeback *wb = list_entry(&bdi->wb_list,
 852                                              struct bdi_writeback, bdi_node);
 853
 854        might_sleep();
 855restart:
 856        rcu_read_lock();
 857        list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) {
 858                DEFINE_WB_COMPLETION(fallback_work_done, bdi);
 859                struct wb_writeback_work fallback_work;
 860                struct wb_writeback_work *work;
 861                long nr_pages;
 862
 863                if (last_wb) {
 864                        wb_put(last_wb);
 865                        last_wb = NULL;
 866                }
 867
 868                /* SYNC_ALL writes out I_DIRTY_TIME too */
 869                if (!wb_has_dirty_io(wb) &&
 870                    (base_work->sync_mode == WB_SYNC_NONE ||
 871                     list_empty(&wb->b_dirty_time)))
 872                        continue;
 873                if (skip_if_busy && writeback_in_progress(wb))
 874                        continue;
 875
 876                nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);
 877
 878                work = kmalloc(sizeof(*work), GFP_ATOMIC);
 879                if (work) {
 880                        *work = *base_work;
 881                        work->nr_pages = nr_pages;
 882                        work->auto_free = 1;
 883                        wb_queue_work(wb, work);
 884                        continue;
 885                }
 886
 887                /* alloc failed, execute synchronously using on-stack fallback */
 888                work = &fallback_work;
 889                *work = *base_work;
 890                work->nr_pages = nr_pages;
 891                work->auto_free = 0;
 892                work->done = &fallback_work_done;
 893
 894                wb_queue_work(wb, work);
 895
 896                /*
 897                 * Pin @wb so that it stays on @bdi->wb_list.  This allows
 898                 * continuing iteration from @wb after dropping and
 899                 * regrabbing rcu read lock.
 900                 */
 901                wb_get(wb);
 902                last_wb = wb;
 903
 904                rcu_read_unlock();
 905                wb_wait_for_completion(&fallback_work_done);
 906                goto restart;
 907        }
 908        rcu_read_unlock();
 909
 910        if (last_wb)
 911                wb_put(last_wb);
 912}
 913
 914/**
 915 * cgroup_writeback_by_id - initiate cgroup writeback from bdi and memcg IDs
 916 * @bdi_id: target bdi id
 917 * @memcg_id: target memcg css id
 918 * @nr: number of pages to write, 0 for best-effort dirty flushing
 919 * @reason: reason why some writeback work initiated
 920 * @done: target wb_completion
 921 *
 922 * Initiate flush of the bdi_writeback identified by @bdi_id and @memcg_id
 923 * with the specified parameters.
 924 */
 925int cgroup_writeback_by_id(u64 bdi_id, int memcg_id, unsigned long nr,
 926                           enum wb_reason reason, struct wb_completion *done)
 927{
 928        struct backing_dev_info *bdi;
 929        struct cgroup_subsys_state *memcg_css;
 930        struct bdi_writeback *wb;
 931        struct wb_writeback_work *work;
 932        int ret;
 933
 934        /* lookup bdi and memcg */
 935        bdi = bdi_get_by_id(bdi_id);
 936        if (!bdi)
 937                return -ENOENT;
 938
 939        rcu_read_lock();
 940        memcg_css = css_from_id(memcg_id, &memory_cgrp_subsys);
 941        if (memcg_css && !css_tryget(memcg_css))
 942                memcg_css = NULL;
 943        rcu_read_unlock();
 944        if (!memcg_css) {
 945                ret = -ENOENT;
 946                goto out_bdi_put;
 947        }
 948
 949        /*
 950         * And find the associated wb.  If the wb isn't there already
 951         * there's nothing to flush, don't create one.
 952         */
 953        wb = wb_get_lookup(bdi, memcg_css);
 954        if (!wb) {
 955                ret = -ENOENT;
 956                goto out_css_put;
 957        }
 958
 959        /*
 960         * If @nr is zero, the caller is attempting to write out most of
 961         * the currently dirty pages.  Let's take the current dirty page
 962         * count and inflate it by 25% which should be large enough to
 963         * flush out most dirty pages while avoiding getting livelocked by
 964         * concurrent dirtiers.
 965         */
 966        if (!nr) {
 967                unsigned long filepages, headroom, dirty, writeback;
 968
 969                mem_cgroup_wb_stats(wb, &filepages, &headroom, &dirty,
 970                                      &writeback);
 971                nr = dirty * 10 / 8;
 972        }
 973
 974        /* issue the writeback work */
 975        work = kzalloc(sizeof(*work), GFP_NOWAIT | __GFP_NOWARN);
 976        if (work) {
 977                work->nr_pages = nr;
 978                work->sync_mode = WB_SYNC_NONE;
 979                work->range_cyclic = 1;
 980                work->reason = reason;
 981                work->done = done;
 982                work->auto_free = 1;
 983                wb_queue_work(wb, work);
 984                ret = 0;
 985        } else {
 986                ret = -ENOMEM;
 987        }
 988
 989        wb_put(wb);
 990out_css_put:
 991        css_put(memcg_css);
 992out_bdi_put:
 993        bdi_put(bdi);
 994        return ret;
 995}
 996
 997/**
 998 * cgroup_writeback_umount - flush inode wb switches for umount
 999 *
1000 * This function is called when a super_block is about to be destroyed and
1001 * flushes in-flight inode wb switches.  An inode wb switch goes through
1002 * RCU and then workqueue, so the two need to be flushed in order to ensure
1003 * that all previously scheduled switches are finished.  As wb switches are
1004 * rare occurrences and synchronize_rcu() can take a while, perform
1005 * flushing iff wb switches are in flight.
1006 */
1007void cgroup_writeback_umount(void)
1008{
1009        if (atomic_read(&isw_nr_in_flight)) {
1010                /*
1011                 * Use rcu_barrier() to wait for all pending callbacks to
1012                 * ensure that all in-flight wb switches are in the workqueue.
1013                 */
1014                rcu_barrier();
1015                flush_workqueue(isw_wq);
1016        }
1017}
1018
1019static int __init cgroup_writeback_init(void)
1020{
1021        isw_wq = alloc_workqueue("inode_switch_wbs", 0, 0);
1022        if (!isw_wq)
1023                return -ENOMEM;
1024        return 0;
1025}
1026fs_initcall(cgroup_writeback_init);
1027
1028#else   /* CONFIG_CGROUP_WRITEBACK */
1029
1030static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }
1031static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }
1032
1033static struct bdi_writeback *
1034locked_inode_to_wb_and_lock_list(struct inode *inode)
1035        __releases(&inode->i_lock)
1036        __acquires(&wb->list_lock)
1037{
1038        struct bdi_writeback *wb = inode_to_wb(inode);
1039
1040        spin_unlock(&inode->i_lock);
1041        spin_lock(&wb->list_lock);
1042        return wb;
1043}
1044
1045static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
1046        __acquires(&wb->list_lock)
1047{
1048        struct bdi_writeback *wb = inode_to_wb(inode);
1049
1050        spin_lock(&wb->list_lock);
1051        return wb;
1052}
1053
1054static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
1055{
1056        return nr_pages;
1057}
1058
1059static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
1060                                  struct wb_writeback_work *base_work,
1061                                  bool skip_if_busy)
1062{
1063        might_sleep();
1064
1065        if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {
1066                base_work->auto_free = 0;
1067                wb_queue_work(&bdi->wb, base_work);
1068        }
1069}
1070
1071#endif  /* CONFIG_CGROUP_WRITEBACK */
1072
1073/*
1074 * Add in the number of potentially dirty inodes, because each inode
1075 * write can dirty pagecache in the underlying blockdev.
1076 */
1077static unsigned long get_nr_dirty_pages(void)
1078{
1079        return global_node_page_state(NR_FILE_DIRTY) +
1080                get_nr_dirty_inodes();
1081}
1082
1083static void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason)
1084{
1085        if (!wb_has_dirty_io(wb))
1086                return;
1087
1088        /*
1089         * All callers of this function want to start writeback of all
1090         * dirty pages. Places like vmscan can call this at a very
1091         * high frequency, causing pointless allocations of tons of
1092         * work items and keeping the flusher threads busy retrieving
1093         * that work. Ensure that we only allow one of them pending and
1094         * inflight at the time.
1095         */
1096        if (test_bit(WB_start_all, &wb->state) ||
1097            test_and_set_bit(WB_start_all, &wb->state))
1098                return;
1099
1100        wb->start_all_reason = reason;
1101        wb_wakeup(wb);
1102}
1103
1104/**
1105 * wb_start_background_writeback - start background writeback
1106 * @wb: bdi_writback to write from
1107 *
1108 * Description:
1109 *   This makes sure WB_SYNC_NONE background writeback happens. When
1110 *   this function returns, it is only guaranteed that for given wb
1111 *   some IO is happening if we are over background dirty threshold.
1112 *   Caller need not hold sb s_umount semaphore.
1113 */
1114void wb_start_background_writeback(struct bdi_writeback *wb)
1115{
1116        /*
1117         * We just wake up the flusher thread. It will perform background
1118         * writeback as soon as there is no other work to do.
1119         */
1120        trace_writeback_wake_background(wb);
1121        wb_wakeup(wb);
1122}
1123
1124/*
1125 * Remove the inode from the writeback list it is on.
1126 */
1127void inode_io_list_del(struct inode *inode)
1128{
1129        struct bdi_writeback *wb;
1130
1131        wb = inode_to_wb_and_lock_list(inode);
1132        spin_lock(&inode->i_lock);
1133        inode_io_list_del_locked(inode, wb);
1134        spin_unlock(&inode->i_lock);
1135        spin_unlock(&wb->list_lock);
1136}
1137EXPORT_SYMBOL(inode_io_list_del);
1138
1139/*
1140 * mark an inode as under writeback on the sb
1141 */
1142void sb_mark_inode_writeback(struct inode *inode)
1143{
1144        struct super_block *sb = inode->i_sb;
1145        unsigned long flags;
1146
1147        if (list_empty(&inode->i_wb_list)) {
1148                spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
1149                if (list_empty(&inode->i_wb_list)) {
1150                        list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb);
1151                        trace_sb_mark_inode_writeback(inode);
1152                }
1153                spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
1154        }
1155}
1156
1157/*
1158 * clear an inode as under writeback on the sb
1159 */
1160void sb_clear_inode_writeback(struct inode *inode)
1161{
1162        struct super_block *sb = inode->i_sb;
1163        unsigned long flags;
1164
1165        if (!list_empty(&inode->i_wb_list)) {
1166                spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
1167                if (!list_empty(&inode->i_wb_list)) {
1168                        list_del_init(&inode->i_wb_list);
1169                        trace_sb_clear_inode_writeback(inode);
1170                }
1171                spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
1172        }
1173}
1174
1175/*
1176 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
1177 * furthest end of its superblock's dirty-inode list.
1178 *
1179 * Before stamping the inode's ->dirtied_when, we check to see whether it is
1180 * already the most-recently-dirtied inode on the b_dirty list.  If that is
1181 * the case then the inode must have been redirtied while it was being written
1182 * out and we don't reset its dirtied_when.
1183 */
1184static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb)
1185{
1186        assert_spin_locked(&inode->i_lock);
1187
1188        if (!list_empty(&wb->b_dirty)) {
1189                struct inode *tail;
1190
1191                tail = wb_inode(wb->b_dirty.next);
1192                if (time_before(inode->dirtied_when, tail->dirtied_when))
1193                        inode->dirtied_when = jiffies;
1194        }
1195        inode_io_list_move_locked(inode, wb, &wb->b_dirty);
1196        inode->i_state &= ~I_SYNC_QUEUED;
1197}
1198
1199static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
1200{
1201        spin_lock(&inode->i_lock);
1202        redirty_tail_locked(inode, wb);
1203        spin_unlock(&inode->i_lock);
1204}
1205
1206/*
1207 * requeue inode for re-scanning after bdi->b_io list is exhausted.
1208 */
1209static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
1210{
1211        inode_io_list_move_locked(inode, wb, &wb->b_more_io);
1212}
1213
1214static void inode_sync_complete(struct inode *inode)
1215{
1216        inode->i_state &= ~I_SYNC;
1217        /* If inode is clean an unused, put it into LRU now... */
1218        inode_add_lru(inode);
1219        /* Waiters must see I_SYNC cleared before being woken up */
1220        smp_mb();
1221        wake_up_bit(&inode->i_state, __I_SYNC);
1222}
1223
1224static bool inode_dirtied_after(struct inode *inode, unsigned long t)
1225{
1226        bool ret = time_after(inode->dirtied_when, t);
1227#ifndef CONFIG_64BIT
1228        /*
1229         * For inodes being constantly redirtied, dirtied_when can get stuck.
1230         * It _appears_ to be in the future, but is actually in distant past.
1231         * This test is necessary to prevent such wrapped-around relative times
1232         * from permanently stopping the whole bdi writeback.
1233         */
1234        ret = ret && time_before_eq(inode->dirtied_when, jiffies);
1235#endif
1236        return ret;
1237}
1238
1239#define EXPIRE_DIRTY_ATIME 0x0001
1240
1241/*
1242 * Move expired (dirtied before dirtied_before) dirty inodes from
1243 * @delaying_queue to @dispatch_queue.
1244 */
1245static int move_expired_inodes(struct list_head *delaying_queue,
1246                               struct list_head *dispatch_queue,
1247                               unsigned long dirtied_before)
1248{
1249        LIST_HEAD(tmp);
1250        struct list_head *pos, *node;
1251        struct super_block *sb = NULL;
1252        struct inode *inode;
1253        int do_sb_sort = 0;
1254        int moved = 0;
1255
1256        while (!list_empty(delaying_queue)) {
1257                inode = wb_inode(delaying_queue->prev);
1258                if (inode_dirtied_after(inode, dirtied_before))
1259                        break;
1260                list_move(&inode->i_io_list, &tmp);
1261                moved++;
1262                spin_lock(&inode->i_lock);
1263                inode->i_state |= I_SYNC_QUEUED;
1264                spin_unlock(&inode->i_lock);
1265                if (sb_is_blkdev_sb(inode->i_sb))
1266                        continue;
1267                if (sb && sb != inode->i_sb)
1268                        do_sb_sort = 1;
1269                sb = inode->i_sb;
1270        }
1271
1272        /* just one sb in list, splice to dispatch_queue and we're done */
1273        if (!do_sb_sort) {
1274                list_splice(&tmp, dispatch_queue);
1275                goto out;
1276        }
1277
1278        /* Move inodes from one superblock together */
1279        while (!list_empty(&tmp)) {
1280                sb = wb_inode(tmp.prev)->i_sb;
1281                list_for_each_prev_safe(pos, node, &tmp) {
1282                        inode = wb_inode(pos);
1283                        if (inode->i_sb == sb)
1284                                list_move(&inode->i_io_list, dispatch_queue);
1285                }
1286        }
1287out:
1288        return moved;
1289}
1290
1291/*
1292 * Queue all expired dirty inodes for io, eldest first.
1293 * Before
1294 *         newly dirtied     b_dirty    b_io    b_more_io
1295 *         =============>    gf         edc     BA
1296 * After
1297 *         newly dirtied     b_dirty    b_io    b_more_io
1298 *         =============>    g          fBAedc
1299 *                                           |
1300 *                                           +--> dequeue for IO
1301 */
1302static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work,
1303                     unsigned long dirtied_before)
1304{
1305        int moved;
1306        unsigned long time_expire_jif = dirtied_before;
1307
1308        assert_spin_locked(&wb->list_lock);
1309        list_splice_init(&wb->b_more_io, &wb->b_io);
1310        moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before);
1311        if (!work->for_sync)
1312                time_expire_jif = jiffies - dirtytime_expire_interval * HZ;
1313        moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,
1314                                     time_expire_jif);
1315        if (moved)
1316                wb_io_lists_populated(wb);
1317        trace_writeback_queue_io(wb, work, dirtied_before, moved);
1318}
1319
1320static int write_inode(struct inode *inode, struct writeback_control *wbc)
1321{
1322        int ret;
1323
1324        if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
1325                trace_writeback_write_inode_start(inode, wbc);
1326                ret = inode->i_sb->s_op->write_inode(inode, wbc);
1327                trace_writeback_write_inode(inode, wbc);
1328                return ret;
1329        }
1330        return 0;
1331}
1332
1333/*
1334 * Wait for writeback on an inode to complete. Called with i_lock held.
1335 * Caller must make sure inode cannot go away when we drop i_lock.
1336 */
1337static void __inode_wait_for_writeback(struct inode *inode)
1338        __releases(inode->i_lock)
1339        __acquires(inode->i_lock)
1340{
1341        DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
1342        wait_queue_head_t *wqh;
1343
1344        wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
1345        while (inode->i_state & I_SYNC) {
1346                spin_unlock(&inode->i_lock);
1347                __wait_on_bit(wqh, &wq, bit_wait,
1348                              TASK_UNINTERRUPTIBLE);
1349                spin_lock(&inode->i_lock);
1350        }
1351}
1352
1353/*
1354 * Wait for writeback on an inode to complete. Caller must have inode pinned.
1355 */
1356void inode_wait_for_writeback(struct inode *inode)
1357{
1358        spin_lock(&inode->i_lock);
1359        __inode_wait_for_writeback(inode);
1360        spin_unlock(&inode->i_lock);
1361}
1362
1363/*
1364 * Sleep until I_SYNC is cleared. This function must be called with i_lock
1365 * held and drops it. It is aimed for callers not holding any inode reference
1366 * so once i_lock is dropped, inode can go away.
1367 */
1368static void inode_sleep_on_writeback(struct inode *inode)
1369        __releases(inode->i_lock)
1370{
1371        DEFINE_WAIT(wait);
1372        wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
1373        int sleep;
1374
1375        prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1376        sleep = inode->i_state & I_SYNC;
1377        spin_unlock(&inode->i_lock);
1378        if (sleep)
1379                schedule();
1380        finish_wait(wqh, &wait);
1381}
1382
1383/*
1384 * Find proper writeback list for the inode depending on its current state and
1385 * possibly also change of its state while we were doing writeback.  Here we
1386 * handle things such as livelock prevention or fairness of writeback among
1387 * inodes. This function can be called only by flusher thread - noone else
1388 * processes all inodes in writeback lists and requeueing inodes behind flusher
1389 * thread's back can have unexpected consequences.
1390 */
1391static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
1392                          struct writeback_control *wbc)
1393{
1394        if (inode->i_state & I_FREEING)
1395                return;
1396
1397        /*
1398         * Sync livelock prevention. Each inode is tagged and synced in one
1399         * shot. If still dirty, it will be redirty_tail()'ed below.  Update
1400         * the dirty time to prevent enqueue and sync it again.
1401         */
1402        if ((inode->i_state & I_DIRTY) &&
1403            (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
1404                inode->dirtied_when = jiffies;
1405
1406        if (wbc->pages_skipped) {
1407                /*
1408                 * writeback is not making progress due to locked
1409                 * buffers. Skip this inode for now.
1410                 */
1411                redirty_tail_locked(inode, wb);
1412                return;
1413        }
1414
1415        if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
1416                /*
1417                 * We didn't write back all the pages.  nfs_writepages()
1418                 * sometimes bales out without doing anything.
1419                 */
1420                if (wbc->nr_to_write <= 0) {
1421                        /* Slice used up. Queue for next turn. */
1422                        requeue_io(inode, wb);
1423                } else {
1424                        /*
1425                         * Writeback blocked by something other than
1426                         * congestion. Delay the inode for some time to
1427                         * avoid spinning on the CPU (100% iowait)
1428                         * retrying writeback of the dirty page/inode
1429                         * that cannot be performed immediately.
1430                         */
1431                        redirty_tail_locked(inode, wb);
1432                }
1433        } else if (inode->i_state & I_DIRTY) {
1434                /*
1435                 * Filesystems can dirty the inode during writeback operations,
1436                 * such as delayed allocation during submission or metadata
1437                 * updates after data IO completion.
1438                 */
1439                redirty_tail_locked(inode, wb);
1440        } else if (inode->i_state & I_DIRTY_TIME) {
1441                inode->dirtied_when = jiffies;
1442                inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);
1443                inode->i_state &= ~I_SYNC_QUEUED;
1444        } else {
1445                /* The inode is clean. Remove from writeback lists. */
1446                inode_io_list_del_locked(inode, wb);
1447        }
1448}
1449
1450/*
1451 * Write out an inode and its dirty pages (or some of its dirty pages, depending
1452 * on @wbc->nr_to_write), and clear the relevant dirty flags from i_state.
1453 *
1454 * This doesn't remove the inode from the writeback list it is on, except
1455 * potentially to move it from b_dirty_time to b_dirty due to timestamp
1456 * expiration.  The caller is otherwise responsible for writeback list handling.
1457 *
1458 * The caller is also responsible for setting the I_SYNC flag beforehand and
1459 * calling inode_sync_complete() to clear it afterwards.
1460 */
1461static int
1462__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
1463{
1464        struct address_space *mapping = inode->i_mapping;
1465        long nr_to_write = wbc->nr_to_write;
1466        unsigned dirty;
1467        int ret;
1468
1469        WARN_ON(!(inode->i_state & I_SYNC));
1470
1471        trace_writeback_single_inode_start(inode, wbc, nr_to_write);
1472
1473        ret = do_writepages(mapping, wbc);
1474
1475        /*
1476         * Make sure to wait on the data before writing out the metadata.
1477         * This is important for filesystems that modify metadata on data
1478         * I/O completion. We don't do it for sync(2) writeback because it has a
1479         * separate, external IO completion path and ->sync_fs for guaranteeing
1480         * inode metadata is written back correctly.
1481         */
1482        if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {
1483                int err = filemap_fdatawait(mapping);
1484                if (ret == 0)
1485                        ret = err;
1486        }
1487
1488        /*
1489         * If the inode has dirty timestamps and we need to write them, call
1490         * mark_inode_dirty_sync() to notify the filesystem about it and to
1491         * change I_DIRTY_TIME into I_DIRTY_SYNC.
1492         */
1493        if ((inode->i_state & I_DIRTY_TIME) &&
1494            (wbc->sync_mode == WB_SYNC_ALL ||
1495             time_after(jiffies, inode->dirtied_time_when +
1496                        dirtytime_expire_interval * HZ))) {
1497                trace_writeback_lazytime(inode);
1498                mark_inode_dirty_sync(inode);
1499        }
1500
1501        /*
1502         * Get and clear the dirty flags from i_state.  This needs to be done
1503         * after calling writepages because some filesystems may redirty the
1504         * inode during writepages due to delalloc.  It also needs to be done
1505         * after handling timestamp expiration, as that may dirty the inode too.
1506         */
1507        spin_lock(&inode->i_lock);
1508        dirty = inode->i_state & I_DIRTY;
1509        inode->i_state &= ~dirty;
1510
1511        /*
1512         * Paired with smp_mb() in __mark_inode_dirty().  This allows
1513         * __mark_inode_dirty() to test i_state without grabbing i_lock -
1514         * either they see the I_DIRTY bits cleared or we see the dirtied
1515         * inode.
1516         *
1517         * I_DIRTY_PAGES is always cleared together above even if @mapping
1518         * still has dirty pages.  The flag is reinstated after smp_mb() if
1519         * necessary.  This guarantees that either __mark_inode_dirty()
1520         * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.
1521         */
1522        smp_mb();
1523
1524        if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
1525                inode->i_state |= I_DIRTY_PAGES;
1526
1527        spin_unlock(&inode->i_lock);
1528
1529        /* Don't write the inode if only I_DIRTY_PAGES was set */
1530        if (dirty & ~I_DIRTY_PAGES) {
1531                int err = write_inode(inode, wbc);
1532                if (ret == 0)
1533                        ret = err;
1534        }
1535        trace_writeback_single_inode(inode, wbc, nr_to_write);
1536        return ret;
1537}
1538
1539/*
1540 * Write out an inode's dirty data and metadata on-demand, i.e. separately from
1541 * the regular batched writeback done by the flusher threads in
1542 * writeback_sb_inodes().  @wbc controls various aspects of the write, such as
1543 * whether it is a data-integrity sync (%WB_SYNC_ALL) or not (%WB_SYNC_NONE).
1544 *
1545 * To prevent the inode from going away, either the caller must have a reference
1546 * to the inode, or the inode must have I_WILL_FREE or I_FREEING set.
1547 */
1548static int writeback_single_inode(struct inode *inode,
1549                                  struct writeback_control *wbc)
1550{
1551        struct bdi_writeback *wb;
1552        int ret = 0;
1553
1554        spin_lock(&inode->i_lock);
1555        if (!atomic_read(&inode->i_count))
1556                WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
1557        else
1558                WARN_ON(inode->i_state & I_WILL_FREE);
1559
1560        if (inode->i_state & I_SYNC) {
1561                /*
1562                 * Writeback is already running on the inode.  For WB_SYNC_NONE,
1563                 * that's enough and we can just return.  For WB_SYNC_ALL, we
1564                 * must wait for the existing writeback to complete, then do
1565                 * writeback again if there's anything left.
1566                 */
1567                if (wbc->sync_mode != WB_SYNC_ALL)
1568                        goto out;
1569                __inode_wait_for_writeback(inode);
1570        }
1571        WARN_ON(inode->i_state & I_SYNC);
1572        /*
1573         * If the inode is already fully clean, then there's nothing to do.
1574         *
1575         * For data-integrity syncs we also need to check whether any pages are
1576         * still under writeback, e.g. due to prior WB_SYNC_NONE writeback.  If
1577         * there are any such pages, we'll need to wait for them.
1578         */
1579        if (!(inode->i_state & I_DIRTY_ALL) &&
1580            (wbc->sync_mode != WB_SYNC_ALL ||
1581             !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
1582                goto out;
1583        inode->i_state |= I_SYNC;
1584        wbc_attach_and_unlock_inode(wbc, inode);
1585
1586        ret = __writeback_single_inode(inode, wbc);
1587
1588        wbc_detach_inode(wbc);
1589
1590        wb = inode_to_wb_and_lock_list(inode);
1591        spin_lock(&inode->i_lock);
1592        /*
1593         * If the inode is now fully clean, then it can be safely removed from
1594         * its writeback list (if any).  Otherwise the flusher threads are
1595         * responsible for the writeback lists.
1596         */
1597        if (!(inode->i_state & I_DIRTY_ALL))
1598                inode_io_list_del_locked(inode, wb);
1599        spin_unlock(&wb->list_lock);
1600        inode_sync_complete(inode);
1601out:
1602        spin_unlock(&inode->i_lock);
1603        return ret;
1604}
1605
1606static long writeback_chunk_size(struct bdi_writeback *wb,
1607                                 struct wb_writeback_work *work)
1608{
1609        long pages;
1610
1611        /*
1612         * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
1613         * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
1614         * here avoids calling into writeback_inodes_wb() more than once.
1615         *
1616         * The intended call sequence for WB_SYNC_ALL writeback is:
1617         *
1618         *      wb_writeback()
1619         *          writeback_sb_inodes()       <== called only once
1620         *              write_cache_pages()     <== called once for each inode
1621         *                   (quickly) tag currently dirty pages
1622         *                   (maybe slowly) sync all tagged pages
1623         */
1624        if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
1625                pages = LONG_MAX;
1626        else {
1627                pages = min(wb->avg_write_bandwidth / 2,
1628                            global_wb_domain.dirty_limit / DIRTY_SCOPE);
1629                pages = min(pages, work->nr_pages);
1630                pages = round_down(pages + MIN_WRITEBACK_PAGES,
1631                                   MIN_WRITEBACK_PAGES);
1632        }
1633
1634        return pages;
1635}
1636
1637/*
1638 * Write a portion of b_io inodes which belong to @sb.
1639 *
1640 * Return the number of pages and/or inodes written.
1641 *
1642 * NOTE! This is called with wb->list_lock held, and will
1643 * unlock and relock that for each inode it ends up doing
1644 * IO for.
1645 */
1646static long writeback_sb_inodes(struct super_block *sb,
1647                                struct bdi_writeback *wb,
1648                                struct wb_writeback_work *work)
1649{
1650        struct writeback_control wbc = {
1651                .sync_mode              = work->sync_mode,
1652                .tagged_writepages      = work->tagged_writepages,
1653                .for_kupdate            = work->for_kupdate,
1654                .for_background         = work->for_background,
1655                .for_sync               = work->for_sync,
1656                .range_cyclic           = work->range_cyclic,
1657                .range_start            = 0,
1658                .range_end              = LLONG_MAX,
1659        };
1660        unsigned long start_time = jiffies;
1661        long write_chunk;
1662        long wrote = 0;  /* count both pages and inodes */
1663
1664        while (!list_empty(&wb->b_io)) {
1665                struct inode *inode = wb_inode(wb->b_io.prev);
1666                struct bdi_writeback *tmp_wb;
1667
1668                if (inode->i_sb != sb) {
1669                        if (work->sb) {
1670                                /*
1671                                 * We only want to write back data for this
1672                                 * superblock, move all inodes not belonging
1673                                 * to it back onto the dirty list.
1674                                 */
1675                                redirty_tail(inode, wb);
1676                                continue;
1677                        }
1678
1679                        /*
1680                         * The inode belongs to a different superblock.
1681                         * Bounce back to the caller to unpin this and
1682                         * pin the next superblock.
1683                         */
1684                        break;
1685                }
1686
1687                /*
1688                 * Don't bother with new inodes or inodes being freed, first
1689                 * kind does not need periodic writeout yet, and for the latter
1690                 * kind writeout is handled by the freer.
1691                 */
1692                spin_lock(&inode->i_lock);
1693                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
1694                        redirty_tail_locked(inode, wb);
1695                        spin_unlock(&inode->i_lock);
1696                        continue;
1697                }
1698                if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
1699                        /*
1700                         * If this inode is locked for writeback and we are not
1701                         * doing writeback-for-data-integrity, move it to
1702                         * b_more_io so that writeback can proceed with the
1703                         * other inodes on s_io.
1704                         *
1705                         * We'll have another go at writing back this inode
1706                         * when we completed a full scan of b_io.
1707                         */
1708                        spin_unlock(&inode->i_lock);
1709                        requeue_io(inode, wb);
1710                        trace_writeback_sb_inodes_requeue(inode);
1711                        continue;
1712                }
1713                spin_unlock(&wb->list_lock);
1714
1715                /*
1716                 * We already requeued the inode if it had I_SYNC set and we
1717                 * are doing WB_SYNC_NONE writeback. So this catches only the
1718                 * WB_SYNC_ALL case.
1719                 */
1720                if (inode->i_state & I_SYNC) {
1721                        /* Wait for I_SYNC. This function drops i_lock... */
1722                        inode_sleep_on_writeback(inode);
1723                        /* Inode may be gone, start again */
1724                        spin_lock(&wb->list_lock);
1725                        continue;
1726                }
1727                inode->i_state |= I_SYNC;
1728                wbc_attach_and_unlock_inode(&wbc, inode);
1729
1730                write_chunk = writeback_chunk_size(wb, work);
1731                wbc.nr_to_write = write_chunk;
1732                wbc.pages_skipped = 0;
1733
1734                /*
1735                 * We use I_SYNC to pin the inode in memory. While it is set
1736                 * evict_inode() will wait so the inode cannot be freed.
1737                 */
1738                __writeback_single_inode(inode, &wbc);
1739
1740                wbc_detach_inode(&wbc);
1741                work->nr_pages -= write_chunk - wbc.nr_to_write;
1742                wrote += write_chunk - wbc.nr_to_write;
1743
1744                if (need_resched()) {
1745                        /*
1746                         * We're trying to balance between building up a nice
1747                         * long list of IOs to improve our merge rate, and
1748                         * getting those IOs out quickly for anyone throttling
1749                         * in balance_dirty_pages().  cond_resched() doesn't
1750                         * unplug, so get our IOs out the door before we
1751                         * give up the CPU.
1752                         */
1753                        blk_flush_plug(current);
1754                        cond_resched();
1755                }
1756
1757                /*
1758                 * Requeue @inode if still dirty.  Be careful as @inode may
1759                 * have been switched to another wb in the meantime.
1760                 */
1761                tmp_wb = inode_to_wb_and_lock_list(inode);
1762                spin_lock(&inode->i_lock);
1763                if (!(inode->i_state & I_DIRTY_ALL))
1764                        wrote++;
1765                requeue_inode(inode, tmp_wb, &wbc);
1766                inode_sync_complete(inode);
1767                spin_unlock(&inode->i_lock);
1768
1769                if (unlikely(tmp_wb != wb)) {
1770                        spin_unlock(&tmp_wb->list_lock);
1771                        spin_lock(&wb->list_lock);
1772                }
1773
1774                /*
1775                 * bail out to wb_writeback() often enough to check
1776                 * background threshold and other termination conditions.
1777                 */
1778                if (wrote) {
1779                        if (time_is_before_jiffies(start_time + HZ / 10UL))
1780                                break;
1781                        if (work->nr_pages <= 0)
1782                                break;
1783                }
1784        }
1785        return wrote;
1786}
1787
1788static long __writeback_inodes_wb(struct bdi_writeback *wb,
1789                                  struct wb_writeback_work *work)
1790{
1791        unsigned long start_time = jiffies;
1792        long wrote = 0;
1793
1794        while (!list_empty(&wb->b_io)) {
1795                struct inode *inode = wb_inode(wb->b_io.prev);
1796                struct super_block *sb = inode->i_sb;
1797
1798                if (!trylock_super(sb)) {
1799                        /*
1800                         * trylock_super() may fail consistently due to
1801                         * s_umount being grabbed by someone else. Don't use
1802                         * requeue_io() to avoid busy retrying the inode/sb.
1803                         */
1804                        redirty_tail(inode, wb);
1805                        continue;
1806                }
1807                wrote += writeback_sb_inodes(sb, wb, work);
1808                up_read(&sb->s_umount);
1809
1810                /* refer to the same tests at the end of writeback_sb_inodes */
1811                if (wrote) {
1812                        if (time_is_before_jiffies(start_time + HZ / 10UL))
1813                                break;
1814                        if (work->nr_pages <= 0)
1815                                break;
1816                }
1817        }
1818        /* Leave any unwritten inodes on b_io */
1819        return wrote;
1820}
1821
1822static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
1823                                enum wb_reason reason)
1824{
1825        struct wb_writeback_work work = {
1826                .nr_pages       = nr_pages,
1827                .sync_mode      = WB_SYNC_NONE,
1828                .range_cyclic   = 1,
1829                .reason         = reason,
1830        };
1831        struct blk_plug plug;
1832
1833        blk_start_plug(&plug);
1834        spin_lock(&wb->list_lock);
1835        if (list_empty(&wb->b_io))
1836                queue_io(wb, &work, jiffies);
1837        __writeback_inodes_wb(wb, &work);
1838        spin_unlock(&wb->list_lock);
1839        blk_finish_plug(&plug);
1840
1841        return nr_pages - work.nr_pages;
1842}
1843
1844/*
1845 * Explicit flushing or periodic writeback of "old" data.
1846 *
1847 * Define "old": the first time one of an inode's pages is dirtied, we mark the
1848 * dirtying-time in the inode's address_space.  So this periodic writeback code
1849 * just walks the superblock inode list, writing back any inodes which are
1850 * older than a specific point in time.
1851 *
1852 * Try to run once per dirty_writeback_interval.  But if a writeback event
1853 * takes longer than a dirty_writeback_interval interval, then leave a
1854 * one-second gap.
1855 *
1856 * dirtied_before takes precedence over nr_to_write.  So we'll only write back
1857 * all dirty pages if they are all attached to "old" mappings.
1858 */
1859static long wb_writeback(struct bdi_writeback *wb,
1860                         struct wb_writeback_work *work)
1861{
1862        unsigned long wb_start = jiffies;
1863        long nr_pages = work->nr_pages;
1864        unsigned long dirtied_before = jiffies;
1865        struct inode *inode;
1866        long progress;
1867        struct blk_plug plug;
1868
1869        blk_start_plug(&plug);
1870        spin_lock(&wb->list_lock);
1871        for (;;) {
1872                /*
1873                 * Stop writeback when nr_pages has been consumed
1874                 */
1875                if (work->nr_pages <= 0)
1876                        break;
1877
1878                /*
1879                 * Background writeout and kupdate-style writeback may
1880                 * run forever. Stop them if there is other work to do
1881                 * so that e.g. sync can proceed. They'll be restarted
1882                 * after the other works are all done.
1883                 */
1884                if ((work->for_background || work->for_kupdate) &&
1885                    !list_empty(&wb->work_list))
1886                        break;
1887
1888                /*
1889                 * For background writeout, stop when we are below the
1890                 * background dirty threshold
1891                 */
1892                if (work->for_background && !wb_over_bg_thresh(wb))
1893                        break;
1894
1895                /*
1896                 * Kupdate and background works are special and we want to
1897                 * include all inodes that need writing. Livelock avoidance is
1898                 * handled by these works yielding to any other work so we are
1899                 * safe.
1900                 */
1901                if (work->for_kupdate) {
1902                        dirtied_before = jiffies -
1903                                msecs_to_jiffies(dirty_expire_interval * 10);
1904                } else if (work->for_background)
1905                        dirtied_before = jiffies;
1906
1907                trace_writeback_start(wb, work);
1908                if (list_empty(&wb->b_io))
1909                        queue_io(wb, work, dirtied_before);
1910                if (work->sb)
1911                        progress = writeback_sb_inodes(work->sb, wb, work);
1912                else
1913                        progress = __writeback_inodes_wb(wb, work);
1914                trace_writeback_written(wb, work);
1915
1916                wb_update_bandwidth(wb, wb_start);
1917
1918                /*
1919                 * Did we write something? Try for more
1920                 *
1921                 * Dirty inodes are moved to b_io for writeback in batches.
1922                 * The completion of the current batch does not necessarily
1923                 * mean the overall work is done. So we keep looping as long
1924                 * as made some progress on cleaning pages or inodes.
1925                 */
1926                if (progress)
1927                        continue;
1928                /*
1929                 * No more inodes for IO, bail
1930                 */
1931                if (list_empty(&wb->b_more_io))
1932                        break;
1933                /*
1934                 * Nothing written. Wait for some inode to
1935                 * become available for writeback. Otherwise
1936                 * we'll just busyloop.
1937                 */
1938                trace_writeback_wait(wb, work);
1939                inode = wb_inode(wb->b_more_io.prev);
1940                spin_lock(&inode->i_lock);
1941                spin_unlock(&wb->list_lock);
1942                /* This function drops i_lock... */
1943                inode_sleep_on_writeback(inode);
1944                spin_lock(&wb->list_lock);
1945        }
1946        spin_unlock(&wb->list_lock);
1947        blk_finish_plug(&plug);
1948
1949        return nr_pages - work->nr_pages;
1950}
1951
1952/*
1953 * Return the next wb_writeback_work struct that hasn't been processed yet.
1954 */
1955static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)
1956{
1957        struct wb_writeback_work *work = NULL;
1958
1959        spin_lock_bh(&wb->work_lock);
1960        if (!list_empty(&wb->work_list)) {
1961                work = list_entry(wb->work_list.next,
1962                                  struct wb_writeback_work, list);
1963                list_del_init(&work->list);
1964        }
1965        spin_unlock_bh(&wb->work_lock);
1966        return work;
1967}
1968
1969static long wb_check_background_flush(struct bdi_writeback *wb)
1970{
1971        if (wb_over_bg_thresh(wb)) {
1972
1973                struct wb_writeback_work work = {
1974                        .nr_pages       = LONG_MAX,
1975                        .sync_mode      = WB_SYNC_NONE,
1976                        .for_background = 1,
1977                        .range_cyclic   = 1,
1978                        .reason         = WB_REASON_BACKGROUND,
1979                };
1980
1981                return wb_writeback(wb, &work);
1982        }
1983
1984        return 0;
1985}
1986
1987static long wb_check_old_data_flush(struct bdi_writeback *wb)
1988{
1989        unsigned long expired;
1990        long nr_pages;
1991
1992        /*
1993         * When set to zero, disable periodic writeback
1994         */
1995        if (!dirty_writeback_interval)
1996                return 0;
1997
1998        expired = wb->last_old_flush +
1999                        msecs_to_jiffies(dirty_writeback_interval * 10);
2000        if (time_before(jiffies, expired))
2001                return 0;
2002
2003        wb->last_old_flush = jiffies;
2004        nr_pages = get_nr_dirty_pages();
2005
2006        if (nr_pages) {
2007                struct wb_writeback_work work = {
2008                        .nr_pages       = nr_pages,
2009                        .sync_mode      = WB_SYNC_NONE,
2010                        .for_kupdate    = 1,
2011                        .range_cyclic   = 1,
2012                        .reason         = WB_REASON_PERIODIC,
2013                };
2014
2015                return wb_writeback(wb, &work);
2016        }
2017
2018        return 0;
2019}
2020
2021static long wb_check_start_all(struct bdi_writeback *wb)
2022{
2023        long nr_pages;
2024
2025        if (!test_bit(WB_start_all, &wb->state))
2026                return 0;
2027
2028        nr_pages = get_nr_dirty_pages();
2029        if (nr_pages) {
2030                struct wb_writeback_work work = {
2031                        .nr_pages       = wb_split_bdi_pages(wb, nr_pages),
2032                        .sync_mode      = WB_SYNC_NONE,
2033                        .range_cyclic   = 1,
2034                        .reason         = wb->start_all_reason,
2035                };
2036
2037                nr_pages = wb_writeback(wb, &work);
2038        }
2039
2040        clear_bit(WB_start_all, &wb->state);
2041        return nr_pages;
2042}
2043
2044
2045/*
2046 * Retrieve work items and do the writeback they describe
2047 */
2048static long wb_do_writeback(struct bdi_writeback *wb)
2049{
2050        struct wb_writeback_work *work;
2051        long wrote = 0;
2052
2053        set_bit(WB_writeback_running, &wb->state);
2054        while ((work = get_next_work_item(wb)) != NULL) {
2055                trace_writeback_exec(wb, work);
2056                wrote += wb_writeback(wb, work);
2057                finish_writeback_work(wb, work);
2058        }
2059
2060        /*
2061         * Check for a flush-everything request
2062         */
2063        wrote += wb_check_start_all(wb);
2064
2065        /*
2066         * Check for periodic writeback, kupdated() style
2067         */
2068        wrote += wb_check_old_data_flush(wb);
2069        wrote += wb_check_background_flush(wb);
2070        clear_bit(WB_writeback_running, &wb->state);
2071
2072        return wrote;
2073}
2074
2075/*
2076 * Handle writeback of dirty data for the device backed by this bdi. Also
2077 * reschedules periodically and does kupdated style flushing.
2078 */
2079void wb_workfn(struct work_struct *work)
2080{
2081        struct bdi_writeback *wb = container_of(to_delayed_work(work),
2082                                                struct bdi_writeback, dwork);
2083        long pages_written;
2084
2085        set_worker_desc("flush-%s", bdi_dev_name(wb->bdi));
2086        current->flags |= PF_SWAPWRITE;
2087
2088        if (likely(!current_is_workqueue_rescuer() ||
2089                   !test_bit(WB_registered, &wb->state))) {
2090                /*
2091                 * The normal path.  Keep writing back @wb until its
2092                 * work_list is empty.  Note that this path is also taken
2093                 * if @wb is shutting down even when we're running off the
2094                 * rescuer as work_list needs to be drained.
2095                 */
2096                do {
2097                        pages_written = wb_do_writeback(wb);
2098                        trace_writeback_pages_written(pages_written);
2099                } while (!list_empty(&wb->work_list));
2100        } else {
2101                /*
2102                 * bdi_wq can't get enough workers and we're running off
2103                 * the emergency worker.  Don't hog it.  Hopefully, 1024 is
2104                 * enough for efficient IO.
2105                 */
2106                pages_written = writeback_inodes_wb(wb, 1024,
2107                                                    WB_REASON_FORKER_THREAD);
2108                trace_writeback_pages_written(pages_written);
2109        }
2110
2111        if (!list_empty(&wb->work_list))
2112                wb_wakeup(wb);
2113        else if (wb_has_dirty_io(wb) && dirty_writeback_interval)
2114                wb_wakeup_delayed(wb);
2115
2116        current->flags &= ~PF_SWAPWRITE;
2117}
2118
2119/*
2120 * Start writeback of `nr_pages' pages on this bdi. If `nr_pages' is zero,
2121 * write back the whole world.
2122 */
2123static void __wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
2124                                         enum wb_reason reason)
2125{
2126        struct bdi_writeback *wb;
2127
2128        if (!bdi_has_dirty_io(bdi))
2129                return;
2130
2131        list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
2132                wb_start_writeback(wb, reason);
2133}
2134
2135void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
2136                                enum wb_reason reason)
2137{
2138        rcu_read_lock();
2139        __wakeup_flusher_threads_bdi(bdi, reason);
2140        rcu_read_unlock();
2141}
2142
2143/*
2144 * Wakeup the flusher threads to start writeback of all currently dirty pages
2145 */
2146void wakeup_flusher_threads(enum wb_reason reason)
2147{
2148        struct backing_dev_info *bdi;
2149
2150        /*
2151         * If we are expecting writeback progress we must submit plugged IO.
2152         */
2153        if (blk_needs_flush_plug(current))
2154                blk_schedule_flush_plug(current);
2155
2156        rcu_read_lock();
2157        list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
2158                __wakeup_flusher_threads_bdi(bdi, reason);
2159        rcu_read_unlock();
2160}
2161
2162/*
2163 * Wake up bdi's periodically to make sure dirtytime inodes gets
2164 * written back periodically.  We deliberately do *not* check the
2165 * b_dirtytime list in wb_has_dirty_io(), since this would cause the
2166 * kernel to be constantly waking up once there are any dirtytime
2167 * inodes on the system.  So instead we define a separate delayed work
2168 * function which gets called much more rarely.  (By default, only
2169 * once every 12 hours.)
2170 *
2171 * If there is any other write activity going on in the file system,
2172 * this function won't be necessary.  But if the only thing that has
2173 * happened on the file system is a dirtytime inode caused by an atime
2174 * update, we need this infrastructure below to make sure that inode
2175 * eventually gets pushed out to disk.
2176 */
2177static void wakeup_dirtytime_writeback(struct work_struct *w);
2178static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);
2179
2180static void wakeup_dirtytime_writeback(struct work_struct *w)
2181{
2182        struct backing_dev_info *bdi;
2183
2184        rcu_read_lock();
2185        list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
2186                struct bdi_writeback *wb;
2187
2188                list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
2189                        if (!list_empty(&wb->b_dirty_time))
2190                                wb_wakeup(wb);
2191        }
2192        rcu_read_unlock();
2193        schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
2194}
2195
2196static int __init start_dirtytime_writeback(void)
2197{
2198        schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
2199        return 0;
2200}
2201__initcall(start_dirtytime_writeback);
2202
2203int dirtytime_interval_handler(struct ctl_table *table, int write,
2204                               void *buffer, size_t *lenp, loff_t *ppos)
2205{
2206        int ret;
2207
2208        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2209        if (ret == 0 && write)
2210                mod_delayed_work(system_wq, &dirtytime_work, 0);
2211        return ret;
2212}
2213
2214/**
2215 * __mark_inode_dirty - internal function to mark an inode dirty
2216 *
2217 * @inode: inode to mark
2218 * @flags: what kind of dirty, e.g. I_DIRTY_SYNC.  This can be a combination of
2219 *         multiple I_DIRTY_* flags, except that I_DIRTY_TIME can't be combined
2220 *         with I_DIRTY_PAGES.
2221 *
2222 * Mark an inode as dirty.  We notify the filesystem, then update the inode's
2223 * dirty flags.  Then, if needed we add the inode to the appropriate dirty list.
2224 *
2225 * Most callers should use mark_inode_dirty() or mark_inode_dirty_sync()
2226 * instead of calling this directly.
2227 *
2228 * CAREFUL!  We only add the inode to the dirty list if it is hashed or if it
2229 * refers to a blockdev.  Unhashed inodes will never be added to the dirty list
2230 * even if they are later hashed, as they will have been marked dirty already.
2231 *
2232 * In short, ensure you hash any inodes _before_ you start marking them dirty.
2233 *
2234 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
2235 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
2236 * the kernel-internal blockdev inode represents the dirtying time of the
2237 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
2238 * page->mapping->host, so the page-dirtying time is recorded in the internal
2239 * blockdev inode.
2240 */
2241void __mark_inode_dirty(struct inode *inode, int flags)
2242{
2243        struct super_block *sb = inode->i_sb;
2244        int dirtytime = 0;
2245
2246        trace_writeback_mark_inode_dirty(inode, flags);
2247
2248        if (flags & I_DIRTY_INODE) {
2249                /*
2250                 * Notify the filesystem about the inode being dirtied, so that
2251                 * (if needed) it can update on-disk fields and journal the
2252                 * inode.  This is only needed when the inode itself is being
2253                 * dirtied now.  I.e. it's only needed for I_DIRTY_INODE, not
2254                 * for just I_DIRTY_PAGES or I_DIRTY_TIME.
2255                 */
2256                trace_writeback_dirty_inode_start(inode, flags);
2257                if (sb->s_op->dirty_inode)
2258                        sb->s_op->dirty_inode(inode, flags & I_DIRTY_INODE);
2259                trace_writeback_dirty_inode(inode, flags);
2260
2261                /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */
2262                flags &= ~I_DIRTY_TIME;
2263        } else {
2264                /*
2265                 * Else it's either I_DIRTY_PAGES, I_DIRTY_TIME, or nothing.
2266                 * (We don't support setting both I_DIRTY_PAGES and I_DIRTY_TIME
2267                 * in one call to __mark_inode_dirty().)
2268                 */
2269                dirtytime = flags & I_DIRTY_TIME;
2270                WARN_ON_ONCE(dirtytime && flags != I_DIRTY_TIME);
2271        }
2272
2273        /*
2274         * Paired with smp_mb() in __writeback_single_inode() for the
2275         * following lockless i_state test.  See there for details.
2276         */
2277        smp_mb();
2278
2279        if (((inode->i_state & flags) == flags) ||
2280            (dirtytime && (inode->i_state & I_DIRTY_INODE)))
2281                return;
2282
2283        spin_lock(&inode->i_lock);
2284        if (dirtytime && (inode->i_state & I_DIRTY_INODE))
2285                goto out_unlock_inode;
2286        if ((inode->i_state & flags) != flags) {
2287                const int was_dirty = inode->i_state & I_DIRTY;
2288
2289                inode_attach_wb(inode, NULL);
2290
2291                /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */
2292                if (flags & I_DIRTY_INODE)
2293                        inode->i_state &= ~I_DIRTY_TIME;
2294                inode->i_state |= flags;
2295
2296                /*
2297                 * If the inode is queued for writeback by flush worker, just
2298                 * update its dirty state. Once the flush worker is done with
2299                 * the inode it will place it on the appropriate superblock
2300                 * list, based upon its state.
2301                 */
2302                if (inode->i_state & I_SYNC_QUEUED)
2303                        goto out_unlock_inode;
2304
2305                /*
2306                 * Only add valid (hashed) inodes to the superblock's
2307                 * dirty list.  Add blockdev inodes as well.
2308                 */
2309                if (!S_ISBLK(inode->i_mode)) {
2310                        if (inode_unhashed(inode))
2311                                goto out_unlock_inode;
2312                }
2313                if (inode->i_state & I_FREEING)
2314                        goto out_unlock_inode;
2315
2316                /*
2317                 * If the inode was already on b_dirty/b_io/b_more_io, don't
2318                 * reposition it (that would break b_dirty time-ordering).
2319                 */
2320                if (!was_dirty) {
2321                        struct bdi_writeback *wb;
2322                        struct list_head *dirty_list;
2323                        bool wakeup_bdi = false;
2324
2325                        wb = locked_inode_to_wb_and_lock_list(inode);
2326
2327                        inode->dirtied_when = jiffies;
2328                        if (dirtytime)
2329                                inode->dirtied_time_when = jiffies;
2330
2331                        if (inode->i_state & I_DIRTY)
2332                                dirty_list = &wb->b_dirty;
2333                        else
2334                                dirty_list = &wb->b_dirty_time;
2335
2336                        wakeup_bdi = inode_io_list_move_locked(inode, wb,
2337                                                               dirty_list);
2338
2339                        spin_unlock(&wb->list_lock);
2340                        trace_writeback_dirty_inode_enqueue(inode);
2341
2342                        /*
2343                         * If this is the first dirty inode for this bdi,
2344                         * we have to wake-up the corresponding bdi thread
2345                         * to make sure background write-back happens
2346                         * later.
2347                         */
2348                        if (wakeup_bdi &&
2349                            (wb->bdi->capabilities & BDI_CAP_WRITEBACK))
2350                                wb_wakeup_delayed(wb);
2351                        return;
2352                }
2353        }
2354out_unlock_inode:
2355        spin_unlock(&inode->i_lock);
2356}
2357EXPORT_SYMBOL(__mark_inode_dirty);
2358
2359/*
2360 * The @s_sync_lock is used to serialise concurrent sync operations
2361 * to avoid lock contention problems with concurrent wait_sb_inodes() calls.
2362 * Concurrent callers will block on the s_sync_lock rather than doing contending
2363 * walks. The queueing maintains sync(2) required behaviour as all the IO that
2364 * has been issued up to the time this function is enter is guaranteed to be
2365 * completed by the time we have gained the lock and waited for all IO that is
2366 * in progress regardless of the order callers are granted the lock.
2367 */
2368static void wait_sb_inodes(struct super_block *sb)
2369{
2370        LIST_HEAD(sync_list);
2371
2372        /*
2373         * We need to be protected against the filesystem going from
2374         * r/o to r/w or vice versa.
2375         */
2376        WARN_ON(!rwsem_is_locked(&sb->s_umount));
2377
2378        mutex_lock(&sb->s_sync_lock);
2379
2380        /*
2381         * Splice the writeback list onto a temporary list to avoid waiting on
2382         * inodes that have started writeback after this point.
2383         *
2384         * Use rcu_read_lock() to keep the inodes around until we have a
2385         * reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as
2386         * the local list because inodes can be dropped from either by writeback
2387         * completion.
2388         */
2389        rcu_read_lock();
2390        spin_lock_irq(&sb->s_inode_wblist_lock);
2391        list_splice_init(&sb->s_inodes_wb, &sync_list);
2392
2393        /*
2394         * Data integrity sync. Must wait for all pages under writeback, because
2395         * there may have been pages dirtied before our sync call, but which had
2396         * writeout started before we write it out.  In which case, the inode
2397         * may not be on the dirty list, but we still have to wait for that
2398         * writeout.
2399         */
2400        while (!list_empty(&sync_list)) {
2401                struct inode *inode = list_first_entry(&sync_list, struct inode,
2402                                                       i_wb_list);
2403                struct address_space *mapping = inode->i_mapping;
2404
2405                /*
2406                 * Move each inode back to the wb list before we drop the lock
2407                 * to preserve consistency between i_wb_list and the mapping
2408                 * writeback tag. Writeback completion is responsible to remove
2409                 * the inode from either list once the writeback tag is cleared.
2410                 */
2411                list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb);
2412
2413                /*
2414                 * The mapping can appear untagged while still on-list since we
2415                 * do not have the mapping lock. Skip it here, wb completion
2416                 * will remove it.
2417                 */
2418                if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
2419                        continue;
2420
2421                spin_unlock_irq(&sb->s_inode_wblist_lock);
2422
2423                spin_lock(&inode->i_lock);
2424                if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) {
2425                        spin_unlock(&inode->i_lock);
2426
2427                        spin_lock_irq(&sb->s_inode_wblist_lock);
2428                        continue;
2429                }
2430                __iget(inode);
2431                spin_unlock(&inode->i_lock);
2432                rcu_read_unlock();
2433
2434                /*
2435                 * We keep the error status of individual mapping so that
2436                 * applications can catch the writeback error using fsync(2).
2437                 * See filemap_fdatawait_keep_errors() for details.
2438                 */
2439                filemap_fdatawait_keep_errors(mapping);
2440
2441                cond_resched();
2442
2443                iput(inode);
2444
2445                rcu_read_lock();
2446                spin_lock_irq(&sb->s_inode_wblist_lock);
2447        }
2448        spin_unlock_irq(&sb->s_inode_wblist_lock);
2449        rcu_read_unlock();
2450        mutex_unlock(&sb->s_sync_lock);
2451}
2452
2453static void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr,
2454                                     enum wb_reason reason, bool skip_if_busy)
2455{
2456        struct backing_dev_info *bdi = sb->s_bdi;
2457        DEFINE_WB_COMPLETION(done, bdi);
2458        struct wb_writeback_work work = {
2459                .sb                     = sb,
2460                .sync_mode              = WB_SYNC_NONE,
2461                .tagged_writepages      = 1,
2462                .done                   = &done,
2463                .nr_pages               = nr,
2464                .reason                 = reason,
2465        };
2466
2467        if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)
2468                return;
2469        WARN_ON(!rwsem_is_locked(&sb->s_umount));
2470
2471        bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy);
2472        wb_wait_for_completion(&done);
2473}
2474
2475/**
2476 * writeback_inodes_sb_nr -     writeback dirty inodes from given super_block
2477 * @sb: the superblock
2478 * @nr: the number of pages to write
2479 * @reason: reason why some writeback work initiated
2480 *
2481 * Start writeback on some inodes on this super_block. No guarantees are made
2482 * on how many (if any) will be written, and this function does not wait
2483 * for IO completion of submitted IO.
2484 */
2485void writeback_inodes_sb_nr(struct super_block *sb,
2486                            unsigned long nr,
2487                            enum wb_reason reason)
2488{
2489        __writeback_inodes_sb_nr(sb, nr, reason, false);
2490}
2491EXPORT_SYMBOL(writeback_inodes_sb_nr);
2492
2493/**
2494 * writeback_inodes_sb  -       writeback dirty inodes from given super_block
2495 * @sb: the superblock
2496 * @reason: reason why some writeback work was initiated
2497 *
2498 * Start writeback on some inodes on this super_block. No guarantees are made
2499 * on how many (if any) will be written, and this function does not wait
2500 * for IO completion of submitted IO.
2501 */
2502void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
2503{
2504        return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
2505}
2506EXPORT_SYMBOL(writeback_inodes_sb);
2507
2508/**
2509 * try_to_writeback_inodes_sb - try to start writeback if none underway
2510 * @sb: the superblock
2511 * @reason: reason why some writeback work was initiated
2512 *
2513 * Invoke __writeback_inodes_sb_nr if no writeback is currently underway.
2514 */
2515void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
2516{
2517        if (!down_read_trylock(&sb->s_umount))
2518                return;
2519
2520        __writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason, true);
2521        up_read(&sb->s_umount);
2522}
2523EXPORT_SYMBOL(try_to_writeback_inodes_sb);
2524
2525/**
2526 * sync_inodes_sb       -       sync sb inode pages
2527 * @sb: the superblock
2528 *
2529 * This function writes and waits on any dirty inode belonging to this
2530 * super_block.
2531 */
2532void sync_inodes_sb(struct super_block *sb)
2533{
2534        struct backing_dev_info *bdi = sb->s_bdi;
2535        DEFINE_WB_COMPLETION(done, bdi);
2536        struct wb_writeback_work work = {
2537                .sb             = sb,
2538                .sync_mode      = WB_SYNC_ALL,
2539                .nr_pages       = LONG_MAX,
2540                .range_cyclic   = 0,
2541                .done           = &done,
2542                .reason         = WB_REASON_SYNC,
2543                .for_sync       = 1,
2544        };
2545
2546        /*
2547         * Can't skip on !bdi_has_dirty() because we should wait for !dirty
2548         * inodes under writeback and I_DIRTY_TIME inodes ignored by
2549         * bdi_has_dirty() need to be written out too.
2550         */
2551        if (bdi == &noop_backing_dev_info)
2552                return;
2553        WARN_ON(!rwsem_is_locked(&sb->s_umount));
2554
2555        /* protect against inode wb switch, see inode_switch_wbs_work_fn() */
2556        bdi_down_write_wb_switch_rwsem(bdi);
2557        bdi_split_work_to_wbs(bdi, &work, false);
2558        wb_wait_for_completion(&done);
2559        bdi_up_write_wb_switch_rwsem(bdi);
2560
2561        wait_sb_inodes(sb);
2562}
2563EXPORT_SYMBOL(sync_inodes_sb);
2564
2565/**
2566 * write_inode_now      -       write an inode to disk
2567 * @inode: inode to write to disk
2568 * @sync: whether the write should be synchronous or not
2569 *
2570 * This function commits an inode to disk immediately if it is dirty. This is
2571 * primarily needed by knfsd.
2572 *
2573 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
2574 */
2575int write_inode_now(struct inode *inode, int sync)
2576{
2577        struct writeback_control wbc = {
2578                .nr_to_write = LONG_MAX,
2579                .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
2580                .range_start = 0,
2581                .range_end = LLONG_MAX,
2582        };
2583
2584        if (!mapping_can_writeback(inode->i_mapping))
2585                wbc.nr_to_write = 0;
2586
2587        might_sleep();
2588        return writeback_single_inode(inode, &wbc);
2589}
2590EXPORT_SYMBOL(write_inode_now);
2591
2592/**
2593 * sync_inode - write an inode and its pages to disk.
2594 * @inode: the inode to sync
2595 * @wbc: controls the writeback mode
2596 *
2597 * sync_inode() will write an inode and its pages to disk.  It will also
2598 * correctly update the inode on its superblock's dirty inode lists and will
2599 * update inode->i_state.
2600 *
2601 * The caller must have a ref on the inode.
2602 */
2603int sync_inode(struct inode *inode, struct writeback_control *wbc)
2604{
2605        return writeback_single_inode(inode, wbc);
2606}
2607EXPORT_SYMBOL(sync_inode);
2608
2609/**
2610 * sync_inode_metadata - write an inode to disk
2611 * @inode: the inode to sync
2612 * @wait: wait for I/O to complete.
2613 *
2614 * Write an inode to disk and adjust its dirty state after completion.
2615 *
2616 * Note: only writes the actual inode, no associated data or other metadata.
2617 */
2618int sync_inode_metadata(struct inode *inode, int wait)
2619{
2620        struct writeback_control wbc = {
2621                .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
2622                .nr_to_write = 0, /* metadata-only */
2623        };
2624
2625        return sync_inode(inode, &wbc);
2626}
2627EXPORT_SYMBOL(sync_inode_metadata);
2628