linux/block/blk-cgroup.h
<<
>>
Prefs
   1#ifndef _BLK_CGROUP_H
   2#define _BLK_CGROUP_H
   3/*
   4 * Common Block IO controller cgroup interface
   5 *
   6 * Based on ideas and code from CFQ, CFS and BFQ:
   7 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
   8 *
   9 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  10 *                    Paolo Valente <paolo.valente@unimore.it>
  11 *
  12 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  13 *                    Nauman Rafique <nauman@google.com>
  14 */
  15
  16#include <linux/cgroup.h>
  17#include <linux/u64_stats_sync.h>
  18#include <linux/seq_file.h>
  19#include <linux/radix-tree.h>
  20#include <linux/blkdev.h>
  21
  22/* Max limits for throttle policy */
  23#define THROTL_IOPS_MAX         UINT_MAX
  24
  25/* CFQ specific, out here for blkcg->cfq_weight */
  26#define CFQ_WEIGHT_MIN          10
  27#define CFQ_WEIGHT_MAX          1000
  28#define CFQ_WEIGHT_DEFAULT      500
  29
  30#ifdef CONFIG_BLK_CGROUP
  31
  32enum blkg_rwstat_type {
  33        BLKG_RWSTAT_READ,
  34        BLKG_RWSTAT_WRITE,
  35        BLKG_RWSTAT_SYNC,
  36        BLKG_RWSTAT_ASYNC,
  37
  38        BLKG_RWSTAT_NR,
  39        BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
  40};
  41
  42struct blkcg_gq;
  43
  44struct blkcg {
  45        struct cgroup_subsys_state      css;
  46        spinlock_t                      lock;
  47
  48        struct radix_tree_root          blkg_tree;
  49        struct blkcg_gq                 *blkg_hint;
  50        struct hlist_head               blkg_list;
  51
  52        /* for policies to test whether associated blkcg has changed */
  53        uint64_t                        id;
  54
  55        /* TODO: per-policy storage in blkcg */
  56        unsigned int                    cfq_weight;     /* belongs to cfq */
  57        unsigned int                    cfq_leaf_weight;
  58};
  59
  60struct blkg_stat {
  61        struct u64_stats_sync           syncp;
  62        uint64_t                        cnt;
  63};
  64
  65struct blkg_rwstat {
  66        struct u64_stats_sync           syncp;
  67        uint64_t                        cnt[BLKG_RWSTAT_NR];
  68};
  69
  70/*
  71 * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
  72 * request_queue (q).  This is used by blkcg policies which need to track
  73 * information per blkcg - q pair.
  74 *
  75 * There can be multiple active blkcg policies and each has its private
  76 * data on each blkg, the size of which is determined by
  77 * blkcg_policy->pd_size.  blkcg core allocates and frees such areas
  78 * together with blkg and invokes pd_init/exit_fn() methods.
  79 *
  80 * Such private data must embed struct blkg_policy_data (pd) at the
  81 * beginning and pd_size can't be smaller than pd.
  82 */
  83struct blkg_policy_data {
  84        /* the blkg and policy id this per-policy data belongs to */
  85        struct blkcg_gq                 *blkg;
  86        int                             plid;
  87
  88        /* used during policy activation */
  89        struct list_head                alloc_node;
  90};
  91
  92/* association between a blk cgroup and a request queue */
  93struct blkcg_gq {
  94        /* Pointer to the associated request_queue */
  95        struct request_queue            *q;
  96        struct list_head                q_node;
  97        struct hlist_node               blkcg_node;
  98        struct blkcg                    *blkcg;
  99
 100        /* all non-root blkcg_gq's are guaranteed to have access to parent */
 101        struct blkcg_gq                 *parent;
 102
 103        /* request allocation list for this blkcg-q pair */
 104        struct request_list             rl;
 105
 106        /* reference count */
 107        int                             refcnt;
 108
 109        /* is this blkg online? protected by both blkcg and q locks */
 110        bool                            online;
 111
 112        struct blkg_policy_data         *pd[BLKCG_MAX_POLS];
 113
 114        struct rcu_head                 rcu_head;
 115};
 116
 117typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
 118typedef void (blkcg_pol_online_pd_fn)(struct blkcg_gq *blkg);
 119typedef void (blkcg_pol_offline_pd_fn)(struct blkcg_gq *blkg);
 120typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg);
 121typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg);
 122
 123struct blkcg_policy {
 124        int                             plid;
 125        /* policy specific private data size */
 126        size_t                          pd_size;
 127        /* cgroup files for the policy */
 128        struct cftype                   *cftypes;
 129
 130        /* operations */
 131        blkcg_pol_init_pd_fn            *pd_init_fn;
 132        blkcg_pol_online_pd_fn          *pd_online_fn;
 133        blkcg_pol_offline_pd_fn         *pd_offline_fn;
 134        blkcg_pol_exit_pd_fn            *pd_exit_fn;
 135        blkcg_pol_reset_pd_stats_fn     *pd_reset_stats_fn;
 136};
 137
 138extern struct blkcg blkcg_root;
 139
 140struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q);
 141struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
 142                                    struct request_queue *q);
 143int blkcg_init_queue(struct request_queue *q);
 144void blkcg_drain_queue(struct request_queue *q);
 145void blkcg_exit_queue(struct request_queue *q);
 146
 147/* Blkio controller policy registration */
 148int blkcg_policy_register(struct blkcg_policy *pol);
 149void blkcg_policy_unregister(struct blkcg_policy *pol);
 150int blkcg_activate_policy(struct request_queue *q,
 151                          const struct blkcg_policy *pol);
 152void blkcg_deactivate_policy(struct request_queue *q,
 153                             const struct blkcg_policy *pol);
 154
 155void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
 156                       u64 (*prfill)(struct seq_file *,
 157                                     struct blkg_policy_data *, int),
 158                       const struct blkcg_policy *pol, int data,
 159                       bool show_total);
 160u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
 161u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 162                         const struct blkg_rwstat *rwstat);
 163u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
 164u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 165                       int off);
 166
 167u64 blkg_stat_recursive_sum(struct blkg_policy_data *pd, int off);
 168struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkg_policy_data *pd,
 169                                             int off);
 170
 171struct blkg_conf_ctx {
 172        struct gendisk                  *disk;
 173        struct blkcg_gq                 *blkg;
 174        u64                             v;
 175};
 176
 177int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
 178                   const char *input, struct blkg_conf_ctx *ctx);
 179void blkg_conf_finish(struct blkg_conf_ctx *ctx);
 180
 181
 182static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
 183{
 184        return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
 185                            struct blkcg, css);
 186}
 187
 188static inline struct blkcg *task_blkcg(struct task_struct *tsk)
 189{
 190        return container_of(task_subsys_state(tsk, blkio_subsys_id),
 191                            struct blkcg, css);
 192}
 193
 194static inline struct blkcg *bio_blkcg(struct bio *bio)
 195{
 196        if (bio && bio->bi_css)
 197                return container_of(bio->bi_css, struct blkcg, css);
 198        return task_blkcg(current);
 199}
 200
 201/**
 202 * blkcg_parent - get the parent of a blkcg
 203 * @blkcg: blkcg of interest
 204 *
 205 * Return the parent blkcg of @blkcg.  Can be called anytime.
 206 */
 207static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
 208{
 209        struct cgroup *pcg = blkcg->css.cgroup->parent;
 210
 211        return pcg ? cgroup_to_blkcg(pcg) : NULL;
 212}
 213
 214/**
 215 * blkg_to_pdata - get policy private data
 216 * @blkg: blkg of interest
 217 * @pol: policy of interest
 218 *
 219 * Return pointer to private data associated with the @blkg-@pol pair.
 220 */
 221static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
 222                                                  struct blkcg_policy *pol)
 223{
 224        return blkg ? blkg->pd[pol->plid] : NULL;
 225}
 226
 227/**
 228 * pdata_to_blkg - get blkg associated with policy private data
 229 * @pd: policy private data of interest
 230 *
 231 * @pd is policy private data.  Determine the blkg it's associated with.
 232 */
 233static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
 234{
 235        return pd ? pd->blkg : NULL;
 236}
 237
 238/**
 239 * blkg_path - format cgroup path of blkg
 240 * @blkg: blkg of interest
 241 * @buf: target buffer
 242 * @buflen: target buffer length
 243 *
 244 * Format the path of the cgroup of @blkg into @buf.
 245 */
 246static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
 247{
 248        int ret;
 249
 250        ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
 251        if (ret)
 252                strncpy(buf, "<unavailable>", buflen);
 253        return ret;
 254}
 255
 256/**
 257 * blkg_get - get a blkg reference
 258 * @blkg: blkg to get
 259 *
 260 * The caller should be holding queue_lock and an existing reference.
 261 */
 262static inline void blkg_get(struct blkcg_gq *blkg)
 263{
 264        lockdep_assert_held(blkg->q->queue_lock);
 265        WARN_ON_ONCE(!blkg->refcnt);
 266        blkg->refcnt++;
 267}
 268
 269void __blkg_release_rcu(struct rcu_head *rcu);
 270
 271/**
 272 * blkg_put - put a blkg reference
 273 * @blkg: blkg to put
 274 *
 275 * The caller should be holding queue_lock.
 276 */
 277static inline void blkg_put(struct blkcg_gq *blkg)
 278{
 279        lockdep_assert_held(blkg->q->queue_lock);
 280        WARN_ON_ONCE(blkg->refcnt <= 0);
 281        if (!--blkg->refcnt)
 282                call_rcu(&blkg->rcu_head, __blkg_release_rcu);
 283}
 284
 285struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, struct request_queue *q,
 286                               bool update_hint);
 287
 288/**
 289 * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
 290 * @d_blkg: loop cursor pointing to the current descendant
 291 * @pos_cgrp: used for iteration
 292 * @p_blkg: target blkg to walk descendants of
 293 *
 294 * Walk @c_blkg through the descendants of @p_blkg.  Must be used with RCU
 295 * read locked.  If called under either blkcg or queue lock, the iteration
 296 * is guaranteed to include all and only online blkgs.  The caller may
 297 * update @pos_cgrp by calling cgroup_rightmost_descendant() to skip
 298 * subtree.
 299 */
 300#define blkg_for_each_descendant_pre(d_blkg, pos_cgrp, p_blkg)          \
 301        cgroup_for_each_descendant_pre((pos_cgrp), (p_blkg)->blkcg->css.cgroup) \
 302                if (((d_blkg) = __blkg_lookup(cgroup_to_blkcg(pos_cgrp), \
 303                                              (p_blkg)->q, false)))
 304
 305/**
 306 * blkg_for_each_descendant_post - post-order walk of a blkg's descendants
 307 * @d_blkg: loop cursor pointing to the current descendant
 308 * @pos_cgrp: used for iteration
 309 * @p_blkg: target blkg to walk descendants of
 310 *
 311 * Similar to blkg_for_each_descendant_pre() but performs post-order
 312 * traversal instead.  Synchronization rules are the same.
 313 */
 314#define blkg_for_each_descendant_post(d_blkg, pos_cgrp, p_blkg)         \
 315        cgroup_for_each_descendant_post((pos_cgrp), (p_blkg)->blkcg->css.cgroup) \
 316                if (((d_blkg) = __blkg_lookup(cgroup_to_blkcg(pos_cgrp), \
 317                                              (p_blkg)->q, false)))
 318
 319/**
 320 * blk_get_rl - get request_list to use
 321 * @q: request_queue of interest
 322 * @bio: bio which will be attached to the allocated request (may be %NULL)
 323 *
 324 * The caller wants to allocate a request from @q to use for @bio.  Find
 325 * the request_list to use and obtain a reference on it.  Should be called
 326 * under queue_lock.  This function is guaranteed to return non-%NULL
 327 * request_list.
 328 */
 329static inline struct request_list *blk_get_rl(struct request_queue *q,
 330                                              struct bio *bio)
 331{
 332        struct blkcg *blkcg;
 333        struct blkcg_gq *blkg;
 334
 335        rcu_read_lock();
 336
 337        blkcg = bio_blkcg(bio);
 338
 339        /* bypass blkg lookup and use @q->root_rl directly for root */
 340        if (blkcg == &blkcg_root)
 341                goto root_rl;
 342
 343        /*
 344         * Try to use blkg->rl.  blkg lookup may fail under memory pressure
 345         * or if either the blkcg or queue is going away.  Fall back to
 346         * root_rl in such cases.
 347         */
 348        blkg = blkg_lookup_create(blkcg, q);
 349        if (unlikely(IS_ERR(blkg)))
 350                goto root_rl;
 351
 352        blkg_get(blkg);
 353        rcu_read_unlock();
 354        return &blkg->rl;
 355root_rl:
 356        rcu_read_unlock();
 357        return &q->root_rl;
 358}
 359
 360/**
 361 * blk_put_rl - put request_list
 362 * @rl: request_list to put
 363 *
 364 * Put the reference acquired by blk_get_rl().  Should be called under
 365 * queue_lock.
 366 */
 367static inline void blk_put_rl(struct request_list *rl)
 368{
 369        /* root_rl may not have blkg set */
 370        if (rl->blkg && rl->blkg->blkcg != &blkcg_root)
 371                blkg_put(rl->blkg);
 372}
 373
 374/**
 375 * blk_rq_set_rl - associate a request with a request_list
 376 * @rq: request of interest
 377 * @rl: target request_list
 378 *
 379 * Associate @rq with @rl so that accounting and freeing can know the
 380 * request_list @rq came from.
 381 */
 382static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl)
 383{
 384        rq->rl = rl;
 385}
 386
 387/**
 388 * blk_rq_rl - return the request_list a request came from
 389 * @rq: request of interest
 390 *
 391 * Return the request_list @rq is allocated from.
 392 */
 393static inline struct request_list *blk_rq_rl(struct request *rq)
 394{
 395        return rq->rl;
 396}
 397
 398struct request_list *__blk_queue_next_rl(struct request_list *rl,
 399                                         struct request_queue *q);
 400/**
 401 * blk_queue_for_each_rl - iterate through all request_lists of a request_queue
 402 *
 403 * Should be used under queue_lock.
 404 */
 405#define blk_queue_for_each_rl(rl, q)    \
 406        for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q)))
 407
 408/**
 409 * blkg_stat_add - add a value to a blkg_stat
 410 * @stat: target blkg_stat
 411 * @val: value to add
 412 *
 413 * Add @val to @stat.  The caller is responsible for synchronizing calls to
 414 * this function.
 415 */
 416static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
 417{
 418        u64_stats_update_begin(&stat->syncp);
 419        stat->cnt += val;
 420        u64_stats_update_end(&stat->syncp);
 421}
 422
 423/**
 424 * blkg_stat_read - read the current value of a blkg_stat
 425 * @stat: blkg_stat to read
 426 *
 427 * Read the current value of @stat.  This function can be called without
 428 * synchroniztion and takes care of u64 atomicity.
 429 */
 430static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
 431{
 432        unsigned int start;
 433        uint64_t v;
 434
 435        do {
 436                start = u64_stats_fetch_begin(&stat->syncp);
 437                v = stat->cnt;
 438        } while (u64_stats_fetch_retry(&stat->syncp, start));
 439
 440        return v;
 441}
 442
 443/**
 444 * blkg_stat_reset - reset a blkg_stat
 445 * @stat: blkg_stat to reset
 446 */
 447static inline void blkg_stat_reset(struct blkg_stat *stat)
 448{
 449        stat->cnt = 0;
 450}
 451
 452/**
 453 * blkg_stat_merge - merge a blkg_stat into another
 454 * @to: the destination blkg_stat
 455 * @from: the source
 456 *
 457 * Add @from's count to @to.
 458 */
 459static inline void blkg_stat_merge(struct blkg_stat *to, struct blkg_stat *from)
 460{
 461        blkg_stat_add(to, blkg_stat_read(from));
 462}
 463
 464/**
 465 * blkg_rwstat_add - add a value to a blkg_rwstat
 466 * @rwstat: target blkg_rwstat
 467 * @rw: mask of REQ_{WRITE|SYNC}
 468 * @val: value to add
 469 *
 470 * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 471 * caller is responsible for synchronizing calls to this function.
 472 */
 473static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
 474                                   int rw, uint64_t val)
 475{
 476        u64_stats_update_begin(&rwstat->syncp);
 477
 478        if (rw & REQ_WRITE)
 479                rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
 480        else
 481                rwstat->cnt[BLKG_RWSTAT_READ] += val;
 482        if (rw & REQ_SYNC)
 483                rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
 484        else
 485                rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
 486
 487        u64_stats_update_end(&rwstat->syncp);
 488}
 489
 490/**
 491 * blkg_rwstat_read - read the current values of a blkg_rwstat
 492 * @rwstat: blkg_rwstat to read
 493 *
 494 * Read the current snapshot of @rwstat and return it as the return value.
 495 * This function can be called without synchronization and takes care of
 496 * u64 atomicity.
 497 */
 498static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
 499{
 500        unsigned int start;
 501        struct blkg_rwstat tmp;
 502
 503        do {
 504                start = u64_stats_fetch_begin(&rwstat->syncp);
 505                tmp = *rwstat;
 506        } while (u64_stats_fetch_retry(&rwstat->syncp, start));
 507
 508        return tmp;
 509}
 510
 511/**
 512 * blkg_rwstat_total - read the total count of a blkg_rwstat
 513 * @rwstat: blkg_rwstat to read
 514 *
 515 * Return the total count of @rwstat regardless of the IO direction.  This
 516 * function can be called without synchronization and takes care of u64
 517 * atomicity.
 518 */
 519static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
 520{
 521        struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
 522
 523        return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
 524}
 525
 526/**
 527 * blkg_rwstat_reset - reset a blkg_rwstat
 528 * @rwstat: blkg_rwstat to reset
 529 */
 530static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
 531{
 532        memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
 533}
 534
 535/**
 536 * blkg_rwstat_merge - merge a blkg_rwstat into another
 537 * @to: the destination blkg_rwstat
 538 * @from: the source
 539 *
 540 * Add @from's counts to @to.
 541 */
 542static inline void blkg_rwstat_merge(struct blkg_rwstat *to,
 543                                     struct blkg_rwstat *from)
 544{
 545        struct blkg_rwstat v = blkg_rwstat_read(from);
 546        int i;
 547
 548        u64_stats_update_begin(&to->syncp);
 549        for (i = 0; i < BLKG_RWSTAT_NR; i++)
 550                to->cnt[i] += v.cnt[i];
 551        u64_stats_update_end(&to->syncp);
 552}
 553
 554#else   /* CONFIG_BLK_CGROUP */
 555
 556struct cgroup;
 557struct blkcg;
 558
 559struct blkg_policy_data {
 560};
 561
 562struct blkcg_gq {
 563};
 564
 565struct blkcg_policy {
 566};
 567
 568static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
 569static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
 570static inline void blkcg_drain_queue(struct request_queue *q) { }
 571static inline void blkcg_exit_queue(struct request_queue *q) { }
 572static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
 573static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
 574static inline int blkcg_activate_policy(struct request_queue *q,
 575                                        const struct blkcg_policy *pol) { return 0; }
 576static inline void blkcg_deactivate_policy(struct request_queue *q,
 577                                           const struct blkcg_policy *pol) { }
 578
 579static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; }
 580static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
 581
 582static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
 583                                                  struct blkcg_policy *pol) { return NULL; }
 584static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
 585static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
 586static inline void blkg_get(struct blkcg_gq *blkg) { }
 587static inline void blkg_put(struct blkcg_gq *blkg) { }
 588
 589static inline struct request_list *blk_get_rl(struct request_queue *q,
 590                                              struct bio *bio) { return &q->root_rl; }
 591static inline void blk_put_rl(struct request_list *rl) { }
 592static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { }
 593static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; }
 594
 595#define blk_queue_for_each_rl(rl, q)    \
 596        for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
 597
 598#endif  /* CONFIG_BLK_CGROUP */
 599#endif  /* _BLK_CGROUP_H */
 600
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.