linux/block/blk-cgroup.h
<<
>>
Prefs
   1#ifndef _BLK_CGROUP_H
   2#define _BLK_CGROUP_H
   3/*
   4 * Common Block IO controller cgroup interface
   5 *
   6 * Based on ideas and code from CFQ, CFS and BFQ:
   7 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
   8 *
   9 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  10 *                    Paolo Valente <paolo.valente@unimore.it>
  11 *
  12 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  13 *                    Nauman Rafique <nauman@google.com>
  14 */
  15
  16#include <linux/cgroup.h>
  17#include <linux/u64_stats_sync.h>
  18#include <linux/seq_file.h>
  19#include <linux/radix-tree.h>
  20#include <linux/blkdev.h>
  21
  22/* Max limits for throttle policy */
  23#define THROTL_IOPS_MAX         UINT_MAX
  24
  25/* CFQ specific, out here for blkcg->cfq_weight */
  26#define CFQ_WEIGHT_MIN          10
  27#define CFQ_WEIGHT_MAX          1000
  28#define CFQ_WEIGHT_DEFAULT      500
  29
  30#ifdef CONFIG_BLK_CGROUP
  31
  32enum blkg_rwstat_type {
  33        BLKG_RWSTAT_READ,
  34        BLKG_RWSTAT_WRITE,
  35        BLKG_RWSTAT_SYNC,
  36        BLKG_RWSTAT_ASYNC,
  37
  38        BLKG_RWSTAT_NR,
  39        BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
  40};
  41
  42struct blkcg_gq;
  43
  44struct blkcg {
  45        struct cgroup_subsys_state      css;
  46        spinlock_t                      lock;
  47
  48        struct radix_tree_root          blkg_tree;
  49        struct blkcg_gq                 *blkg_hint;
  50        struct hlist_head               blkg_list;
  51
  52        /* for policies to test whether associated blkcg has changed */
  53        uint64_t                        id;
  54
  55        /* TODO: per-policy storage in blkcg */
  56        unsigned int                    cfq_weight;     /* belongs to cfq */
  57        unsigned int                    cfq_leaf_weight;
  58};
  59
  60struct blkg_stat {
  61        struct u64_stats_sync           syncp;
  62        uint64_t                        cnt;
  63};
  64
  65struct blkg_rwstat {
  66        struct u64_stats_sync           syncp;
  67        uint64_t                        cnt[BLKG_RWSTAT_NR];
  68};
  69
  70/*
  71 * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
  72 * request_queue (q).  This is used by blkcg policies which need to track
  73 * information per blkcg - q pair.
  74 *
  75 * There can be multiple active blkcg policies and each has its private
  76 * data on each blkg, the size of which is determined by
  77 * blkcg_policy->pd_size.  blkcg core allocates and frees such areas
  78 * together with blkg and invokes pd_init/exit_fn() methods.
  79 *
  80 * Such private data must embed struct blkg_policy_data (pd) at the
  81 * beginning and pd_size can't be smaller than pd.
  82 */
  83struct blkg_policy_data {
  84        /* the blkg and policy id this per-policy data belongs to */
  85        struct blkcg_gq                 *blkg;
  86        int                             plid;
  87
  88        /* used during policy activation */
  89        struct list_head                alloc_node;
  90};
  91
  92/* association between a blk cgroup and a request queue */
  93struct blkcg_gq {
  94        /* Pointer to the associated request_queue */
  95        struct request_queue            *q;
  96        struct list_head                q_node;
  97        struct hlist_node               blkcg_node;
  98        struct blkcg                    *blkcg;
  99
 100        /* all non-root blkcg_gq's are guaranteed to have access to parent */
 101        struct blkcg_gq                 *parent;
 102
 103        /* request allocation list for this blkcg-q pair */
 104        struct request_list             rl;
 105
 106        /* reference count */
 107        int                             refcnt;
 108
 109        /* is this blkg online? protected by both blkcg and q locks */
 110        bool                            online;
 111
 112        struct blkg_policy_data         *pd[BLKCG_MAX_POLS];
 113
 114        struct rcu_head                 rcu_head;
 115};
 116
 117typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
 118typedef void (blkcg_pol_online_pd_fn)(struct blkcg_gq *blkg);
 119typedef void (blkcg_pol_offline_pd_fn)(struct blkcg_gq *blkg);
 120typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg);
 121typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg);
 122
 123struct blkcg_policy {
 124        int                             plid;
 125        /* policy specific private data size */
 126        size_t                          pd_size;
 127        /* cgroup files for the policy */
 128        struct cftype                   *cftypes;
 129
 130        /* operations */
 131        blkcg_pol_init_pd_fn            *pd_init_fn;
 132        blkcg_pol_online_pd_fn          *pd_online_fn;
 133        blkcg_pol_offline_pd_fn         *pd_offline_fn;
 134        blkcg_pol_exit_pd_fn            *pd_exit_fn;
 135        blkcg_pol_reset_pd_stats_fn     *pd_reset_stats_fn;
 136};
 137
 138extern struct blkcg blkcg_root;
 139
 140struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q);
 141struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
 142                                    struct request_queue *q);
 143int blkcg_init_queue(struct request_queue *q);
 144void blkcg_drain_queue(struct request_queue *q);
 145void blkcg_exit_queue(struct request_queue *q);
 146
 147/* Blkio controller policy registration */
 148int blkcg_policy_register(struct blkcg_policy *pol);
 149void blkcg_policy_unregister(struct blkcg_policy *pol);
 150int blkcg_activate_policy(struct request_queue *q,
 151                          const struct blkcg_policy *pol);
 152void blkcg_deactivate_policy(struct request_queue *q,
 153                             const struct blkcg_policy *pol);
 154
 155void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
 156                       u64 (*prfill)(struct seq_file *,
 157                                     struct blkg_policy_data *, int),
 158                       const struct blkcg_policy *pol, int data,
 159                       bool show_total);
 160u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
 161u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 162                         const struct blkg_rwstat *rwstat);
 163u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
 164u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 165                       int off);
 166
 167u64 blkg_stat_recursive_sum(struct blkg_policy_data *pd, int off);
 168struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkg_policy_data *pd,
 169                                             int off);
 170
 171struct blkg_conf_ctx {
 172        struct gendisk                  *disk;
 173        struct blkcg_gq                 *blkg;
 174        u64                             v;
 175};
 176
 177int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
 178                   const char *input, struct blkg_conf_ctx *ctx);
 179void blkg_conf_finish(struct blkg_conf_ctx *ctx);
 180
 181
 182static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
 183{
 184        return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
 185                            struct blkcg, css);
 186}
 187
 188static inline struct blkcg *task_blkcg(struct task_struct *tsk)
 189{
 190        return container_of(task_subsys_state(tsk, blkio_subsys_id),
 191                            struct blkcg, css);
 192}
 193
 194static inline struct blkcg *bio_blkcg(struct bio *bio)
 195{
 196        if (bio && bio->bi_css)
 197                return container_of(bio->bi_css, struct blkcg, css);
 198        return task_blkcg(current);
 199}
 200
 201/**
 202 * blkcg_parent - get the parent of a blkcg
 203 * @blkcg: blkcg of interest
 204 *
 205 * Return the parent blkcg of @blkcg.  Can be called anytime.
 206 */
 207static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
 208{
 209        struct cgroup *pcg = blkcg->css.cgroup->parent;
 210
 211        return pcg ? cgroup_to_blkcg(pcg) : NULL;
 212}
 213
 214/**
 215 * blkg_to_pdata - get policy private data
 216 * @blkg: blkg of interest
 217 * @pol: policy of interest
 218 *
 219 * Return pointer to private data associated with the @blkg-@pol pair.
 220 */
 221static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
 222                                                  struct blkcg_policy *pol)
 223{
 224        return blkg ? blkg->pd[pol->plid] : NULL;
 225}
 226
 227/**
 228 * pdata_to_blkg - get blkg associated with policy private data
 229 * @pd: policy private data of interest
 230 *
 231 * @pd is policy private data.  Determine the blkg it's associated with.
 232 */
 233static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
 234{
 235        return pd ? pd->blkg : NULL;
 236}
 237
 238/**
 239 * blkg_path - format cgroup path of blkg
 240 * @blkg: blkg of interest
 241 * @buf: target buffer
 242 * @buflen: target buffer length
 243 *
 244 * Format the path of the cgroup of @blkg into @buf.
 245 */
 246static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
 247{
 248        int ret;
 249
 250        rcu_read_lock();
 251        ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
 252        rcu_read_unlock();
 253        if (ret)
 254                strncpy(buf, "<unavailable>", buflen);
 255        return ret;
 256}
 257
 258/**
 259 * blkg_get - get a blkg reference
 260 * @blkg: blkg to get
 261 *
 262 * The caller should be holding queue_lock and an existing reference.
 263 */
 264static inline void blkg_get(struct blkcg_gq *blkg)
 265{
 266        lockdep_assert_held(blkg->q->queue_lock);
 267        WARN_ON_ONCE(!blkg->refcnt);
 268        blkg->refcnt++;
 269}
 270
 271void __blkg_release(struct blkcg_gq *blkg);
 272
 273/**
 274 * blkg_put - put a blkg reference
 275 * @blkg: blkg to put
 276 *
 277 * The caller should be holding queue_lock.
 278 */
 279static inline void blkg_put(struct blkcg_gq *blkg)
 280{
 281        lockdep_assert_held(blkg->q->queue_lock);
 282        WARN_ON_ONCE(blkg->refcnt <= 0);
 283        if (!--blkg->refcnt)
 284                __blkg_release(blkg);
 285}
 286
 287/**
 288 * blk_get_rl - get request_list to use
 289 * @q: request_queue of interest
 290 * @bio: bio which will be attached to the allocated request (may be %NULL)
 291 *
 292 * The caller wants to allocate a request from @q to use for @bio.  Find
 293 * the request_list to use and obtain a reference on it.  Should be called
 294 * under queue_lock.  This function is guaranteed to return non-%NULL
 295 * request_list.
 296 */
 297static inline struct request_list *blk_get_rl(struct request_queue *q,
 298                                              struct bio *bio)
 299{
 300        struct blkcg *blkcg;
 301        struct blkcg_gq *blkg;
 302
 303        rcu_read_lock();
 304
 305        blkcg = bio_blkcg(bio);
 306
 307        /* bypass blkg lookup and use @q->root_rl directly for root */
 308        if (blkcg == &blkcg_root)
 309                goto root_rl;
 310
 311        /*
 312         * Try to use blkg->rl.  blkg lookup may fail under memory pressure
 313         * or if either the blkcg or queue is going away.  Fall back to
 314         * root_rl in such cases.
 315         */
 316        blkg = blkg_lookup_create(blkcg, q);
 317        if (unlikely(IS_ERR(blkg)))
 318                goto root_rl;
 319
 320        blkg_get(blkg);
 321        rcu_read_unlock();
 322        return &blkg->rl;
 323root_rl:
 324        rcu_read_unlock();
 325        return &q->root_rl;
 326}
 327
 328/**
 329 * blk_put_rl - put request_list
 330 * @rl: request_list to put
 331 *
 332 * Put the reference acquired by blk_get_rl().  Should be called under
 333 * queue_lock.
 334 */
 335static inline void blk_put_rl(struct request_list *rl)
 336{
 337        /* root_rl may not have blkg set */
 338        if (rl->blkg && rl->blkg->blkcg != &blkcg_root)
 339                blkg_put(rl->blkg);
 340}
 341
 342/**
 343 * blk_rq_set_rl - associate a request with a request_list
 344 * @rq: request of interest
 345 * @rl: target request_list
 346 *
 347 * Associate @rq with @rl so that accounting and freeing can know the
 348 * request_list @rq came from.
 349 */
 350static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl)
 351{
 352        rq->rl = rl;
 353}
 354
 355/**
 356 * blk_rq_rl - return the request_list a request came from
 357 * @rq: request of interest
 358 *
 359 * Return the request_list @rq is allocated from.
 360 */
 361static inline struct request_list *blk_rq_rl(struct request *rq)
 362{
 363        return rq->rl;
 364}
 365
 366struct request_list *__blk_queue_next_rl(struct request_list *rl,
 367                                         struct request_queue *q);
 368/**
 369 * blk_queue_for_each_rl - iterate through all request_lists of a request_queue
 370 *
 371 * Should be used under queue_lock.
 372 */
 373#define blk_queue_for_each_rl(rl, q)    \
 374        for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q)))
 375
 376/**
 377 * blkg_stat_add - add a value to a blkg_stat
 378 * @stat: target blkg_stat
 379 * @val: value to add
 380 *
 381 * Add @val to @stat.  The caller is responsible for synchronizing calls to
 382 * this function.
 383 */
 384static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
 385{
 386        u64_stats_update_begin(&stat->syncp);
 387        stat->cnt += val;
 388        u64_stats_update_end(&stat->syncp);
 389}
 390
 391/**
 392 * blkg_stat_read - read the current value of a blkg_stat
 393 * @stat: blkg_stat to read
 394 *
 395 * Read the current value of @stat.  This function can be called without
 396 * synchroniztion and takes care of u64 atomicity.
 397 */
 398static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
 399{
 400        unsigned int start;
 401        uint64_t v;
 402
 403        do {
 404                start = u64_stats_fetch_begin(&stat->syncp);
 405                v = stat->cnt;
 406        } while (u64_stats_fetch_retry(&stat->syncp, start));
 407
 408        return v;
 409}
 410
 411/**
 412 * blkg_stat_reset - reset a blkg_stat
 413 * @stat: blkg_stat to reset
 414 */
 415static inline void blkg_stat_reset(struct blkg_stat *stat)
 416{
 417        stat->cnt = 0;
 418}
 419
 420/**
 421 * blkg_stat_merge - merge a blkg_stat into another
 422 * @to: the destination blkg_stat
 423 * @from: the source
 424 *
 425 * Add @from's count to @to.
 426 */
 427static inline void blkg_stat_merge(struct blkg_stat *to, struct blkg_stat *from)
 428{
 429        blkg_stat_add(to, blkg_stat_read(from));
 430}
 431
 432/**
 433 * blkg_rwstat_add - add a value to a blkg_rwstat
 434 * @rwstat: target blkg_rwstat
 435 * @rw: mask of REQ_{WRITE|SYNC}
 436 * @val: value to add
 437 *
 438 * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 439 * caller is responsible for synchronizing calls to this function.
 440 */
 441static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
 442                                   int rw, uint64_t val)
 443{
 444        u64_stats_update_begin(&rwstat->syncp);
 445
 446        if (rw & REQ_WRITE)
 447                rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
 448        else
 449                rwstat->cnt[BLKG_RWSTAT_READ] += val;
 450        if (rw & REQ_SYNC)
 451                rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
 452        else
 453                rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
 454
 455        u64_stats_update_end(&rwstat->syncp);
 456}
 457
 458/**
 459 * blkg_rwstat_read - read the current values of a blkg_rwstat
 460 * @rwstat: blkg_rwstat to read
 461 *
 462 * Read the current snapshot of @rwstat and return it as the return value.
 463 * This function can be called without synchronization and takes care of
 464 * u64 atomicity.
 465 */
 466static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
 467{
 468        unsigned int start;
 469        struct blkg_rwstat tmp;
 470
 471        do {
 472                start = u64_stats_fetch_begin(&rwstat->syncp);
 473                tmp = *rwstat;
 474        } while (u64_stats_fetch_retry(&rwstat->syncp, start));
 475
 476        return tmp;
 477}
 478
 479/**
 480 * blkg_rwstat_total - read the total count of a blkg_rwstat
 481 * @rwstat: blkg_rwstat to read
 482 *
 483 * Return the total count of @rwstat regardless of the IO direction.  This
 484 * function can be called without synchronization and takes care of u64
 485 * atomicity.
 486 */
 487static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
 488{
 489        struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
 490
 491        return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
 492}
 493
 494/**
 495 * blkg_rwstat_reset - reset a blkg_rwstat
 496 * @rwstat: blkg_rwstat to reset
 497 */
 498static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
 499{
 500        memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
 501}
 502
 503/**
 504 * blkg_rwstat_merge - merge a blkg_rwstat into another
 505 * @to: the destination blkg_rwstat
 506 * @from: the source
 507 *
 508 * Add @from's counts to @to.
 509 */
 510static inline void blkg_rwstat_merge(struct blkg_rwstat *to,
 511                                     struct blkg_rwstat *from)
 512{
 513        struct blkg_rwstat v = blkg_rwstat_read(from);
 514        int i;
 515
 516        u64_stats_update_begin(&to->syncp);
 517        for (i = 0; i < BLKG_RWSTAT_NR; i++)
 518                to->cnt[i] += v.cnt[i];
 519        u64_stats_update_end(&to->syncp);
 520}
 521
 522#else   /* CONFIG_BLK_CGROUP */
 523
 524struct cgroup;
 525struct blkcg;
 526
 527struct blkg_policy_data {
 528};
 529
 530struct blkcg_gq {
 531};
 532
 533struct blkcg_policy {
 534};
 535
 536static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
 537static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
 538static inline void blkcg_drain_queue(struct request_queue *q) { }
 539static inline void blkcg_exit_queue(struct request_queue *q) { }
 540static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
 541static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
 542static inline int blkcg_activate_policy(struct request_queue *q,
 543                                        const struct blkcg_policy *pol) { return 0; }
 544static inline void blkcg_deactivate_policy(struct request_queue *q,
 545                                           const struct blkcg_policy *pol) { }
 546
 547static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; }
 548static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
 549
 550static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
 551                                                  struct blkcg_policy *pol) { return NULL; }
 552static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
 553static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
 554static inline void blkg_get(struct blkcg_gq *blkg) { }
 555static inline void blkg_put(struct blkcg_gq *blkg) { }
 556
 557static inline struct request_list *blk_get_rl(struct request_queue *q,
 558                                              struct bio *bio) { return &q->root_rl; }
 559static inline void blk_put_rl(struct request_list *rl) { }
 560static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { }
 561static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; }
 562
 563#define blk_queue_for_each_rl(rl, q)    \
 564        for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)
 565
 566#endif  /* CONFIG_BLK_CGROUP */
 567#endif  /* _BLK_CGROUP_H */
 568
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.