linux/kernel/cgroup.c
<<
>>
Prefs
   1/*
   2 *  Generic process-grouping system.
   3 *
   4 *  Based originally on the cpuset system, extracted by Paul Menage
   5 *  Copyright (C) 2006 Google, Inc
   6 *
   7 *  Notifications support
   8 *  Copyright (C) 2009 Nokia Corporation
   9 *  Author: Kirill A. Shutemov
  10 *
  11 *  Copyright notices from the original cpuset code:
  12 *  --------------------------------------------------
  13 *  Copyright (C) 2003 BULL SA.
  14 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
  15 *
  16 *  Portions derived from Patrick Mochel's sysfs code.
  17 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
  18 *
  19 *  2003-10-10 Written by Simon Derr.
  20 *  2003-10-22 Updates by Stephen Hemminger.
  21 *  2004 May-July Rework by Paul Jackson.
  22 *  ---------------------------------------------------
  23 *
  24 *  This file is subject to the terms and conditions of the GNU General Public
  25 *  License.  See the file COPYING in the main directory of the Linux
  26 *  distribution for more details.
  27 */
  28
  29#include <linux/cgroup.h>
  30#include <linux/cred.h>
  31#include <linux/ctype.h>
  32#include <linux/errno.h>
  33#include <linux/init_task.h>
  34#include <linux/kernel.h>
  35#include <linux/list.h>
  36#include <linux/mm.h>
  37#include <linux/mutex.h>
  38#include <linux/mount.h>
  39#include <linux/pagemap.h>
  40#include <linux/proc_fs.h>
  41#include <linux/rcupdate.h>
  42#include <linux/sched.h>
  43#include <linux/backing-dev.h>
  44#include <linux/seq_file.h>
  45#include <linux/slab.h>
  46#include <linux/magic.h>
  47#include <linux/spinlock.h>
  48#include <linux/string.h>
  49#include <linux/sort.h>
  50#include <linux/kmod.h>
  51#include <linux/module.h>
  52#include <linux/delayacct.h>
  53#include <linux/cgroupstats.h>
  54#include <linux/hashtable.h>
  55#include <linux/namei.h>
  56#include <linux/pid_namespace.h>
  57#include <linux/idr.h>
  58#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
  59#include <linux/eventfd.h>
  60#include <linux/poll.h>
  61#include <linux/flex_array.h> /* used in cgroup_attach_task */
  62#include <linux/kthread.h>
  63#include <linux/file.h>
  64
  65#include <linux/atomic.h>
  66
  67/*
  68 * cgroup_mutex is the master lock.  Any modification to cgroup or its
  69 * hierarchy must be performed while holding it.
  70 *
  71 * cgroup_root_mutex nests inside cgroup_mutex and should be held to modify
  72 * cgroupfs_root of any cgroup hierarchy - subsys list, flags,
  73 * release_agent_path and so on.  Modifying requires both cgroup_mutex and
  74 * cgroup_root_mutex.  Readers can acquire either of the two.  This is to
  75 * break the following locking order cycle.
  76 *
  77 *  A. cgroup_mutex -> cred_guard_mutex -> s_type->i_mutex_key -> namespace_sem
  78 *  B. namespace_sem -> cgroup_mutex
  79 *
  80 * B happens only through cgroup_show_options() and using cgroup_root_mutex
  81 * breaks it.
  82 */
  83#ifdef CONFIG_PROVE_RCU
  84DEFINE_MUTEX(cgroup_mutex);
  85EXPORT_SYMBOL_GPL(cgroup_mutex);        /* only for lockdep */
  86#else
  87static DEFINE_MUTEX(cgroup_mutex);
  88#endif
  89
  90static DEFINE_MUTEX(cgroup_root_mutex);
  91
  92/*
  93 * Generate an array of cgroup subsystem pointers. At boot time, this is
  94 * populated with the built in subsystems, and modular subsystems are
  95 * registered after that. The mutable section of this array is protected by
  96 * cgroup_mutex.
  97 */
  98#define SUBSYS(_x) [_x ## _subsys_id] = &_x ## _subsys,
  99#define IS_SUBSYS_ENABLED(option) IS_BUILTIN(option)
 100static struct cgroup_subsys *cgroup_subsys[CGROUP_SUBSYS_COUNT] = {
 101#include <linux/cgroup_subsys.h>
 102};
 103
 104/*
 105 * The dummy hierarchy, reserved for the subsystems that are otherwise
 106 * unattached - it never has more than a single cgroup, and all tasks are
 107 * part of that cgroup.
 108 */
 109static struct cgroupfs_root cgroup_dummy_root;
 110
 111/* dummy_top is a shorthand for the dummy hierarchy's top cgroup */
 112static struct cgroup * const cgroup_dummy_top = &cgroup_dummy_root.top_cgroup;
 113
 114/*
 115 * cgroupfs file entry, pointed to from leaf dentry->d_fsdata.
 116 */
 117struct cfent {
 118        struct list_head                node;
 119        struct dentry                   *dentry;
 120        struct cftype                   *type;
 121        struct cgroup_subsys_state      *css;
 122
 123        /* file xattrs */
 124        struct simple_xattrs            xattrs;
 125};
 126
 127/*
 128 * CSS ID -- ID per subsys's Cgroup Subsys State(CSS). used only when
 129 * cgroup_subsys->use_id != 0.
 130 */
 131#define CSS_ID_MAX      (65535)
 132struct css_id {
 133        /*
 134         * The css to which this ID points. This pointer is set to valid value
 135         * after cgroup is populated. If cgroup is removed, this will be NULL.
 136         * This pointer is expected to be RCU-safe because destroy()
 137         * is called after synchronize_rcu(). But for safe use, css_tryget()
 138         * should be used for avoiding race.
 139         */
 140        struct cgroup_subsys_state __rcu *css;
 141        /*
 142         * ID of this css.
 143         */
 144        unsigned short id;
 145        /*
 146         * Depth in hierarchy which this ID belongs to.
 147         */
 148        unsigned short depth;
 149        /*
 150         * ID is freed by RCU. (and lookup routine is RCU safe.)
 151         */
 152        struct rcu_head rcu_head;
 153        /*
 154         * Hierarchy of CSS ID belongs to.
 155         */
 156        unsigned short stack[0]; /* Array of Length (depth+1) */
 157};
 158
 159/*
 160 * cgroup_event represents events which userspace want to receive.
 161 */
 162struct cgroup_event {
 163        /*
 164         * css which the event belongs to.
 165         */
 166        struct cgroup_subsys_state *css;
 167        /*
 168         * Control file which the event associated.
 169         */
 170        struct cftype *cft;
 171        /*
 172         * eventfd to signal userspace about the event.
 173         */
 174        struct eventfd_ctx *eventfd;
 175        /*
 176         * Each of these stored in a list by the cgroup.
 177         */
 178        struct list_head list;
 179        /*
 180         * All fields below needed to unregister event when
 181         * userspace closes eventfd.
 182         */
 183        poll_table pt;
 184        wait_queue_head_t *wqh;
 185        wait_queue_t wait;
 186        struct work_struct remove;
 187};
 188
 189/* The list of hierarchy roots */
 190
 191static LIST_HEAD(cgroup_roots);
 192static int cgroup_root_count;
 193
 194/*
 195 * Hierarchy ID allocation and mapping.  It follows the same exclusion
 196 * rules as other root ops - both cgroup_mutex and cgroup_root_mutex for
 197 * writes, either for reads.
 198 */
 199static DEFINE_IDR(cgroup_hierarchy_idr);
 200
 201static struct cgroup_name root_cgroup_name = { .name = "/" };
 202
 203/*
 204 * Assign a monotonically increasing serial number to cgroups.  It
 205 * guarantees cgroups with bigger numbers are newer than those with smaller
 206 * numbers.  Also, as cgroups are always appended to the parent's
 207 * ->children list, it guarantees that sibling cgroups are always sorted in
 208 * the ascending serial number order on the list.  Protected by
 209 * cgroup_mutex.
 210 */
 211static u64 cgroup_serial_nr_next = 1;
 212
 213/* This flag indicates whether tasks in the fork and exit paths should
 214 * check for fork/exit handlers to call. This avoids us having to do
 215 * extra work in the fork/exit path if none of the subsystems need to
 216 * be called.
 217 */
 218static int need_forkexit_callback __read_mostly;
 219
 220static struct cftype cgroup_base_files[];
 221
 222static void cgroup_destroy_css_killed(struct cgroup *cgrp);
 223static int cgroup_destroy_locked(struct cgroup *cgrp);
 224static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
 225                              bool is_add);
 226
 227/**
 228 * cgroup_css - obtain a cgroup's css for the specified subsystem
 229 * @cgrp: the cgroup of interest
 230 * @ss: the subsystem of interest (%NULL returns the dummy_css)
 231 *
 232 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 233 * function must be called either under cgroup_mutex or rcu_read_lock() and
 234 * the caller is responsible for pinning the returned css if it wants to
 235 * keep accessing it outside the said locks.  This function may return
 236 * %NULL if @cgrp doesn't have @subsys_id enabled.
 237 */
 238static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
 239                                              struct cgroup_subsys *ss)
 240{
 241        if (ss)
 242                return rcu_dereference_check(cgrp->subsys[ss->subsys_id],
 243                                             lockdep_is_held(&cgroup_mutex));
 244        else
 245                return &cgrp->dummy_css;
 246}
 247
 248/* convenient tests for these bits */
 249static inline bool cgroup_is_dead(const struct cgroup *cgrp)
 250{
 251        return test_bit(CGRP_DEAD, &cgrp->flags);
 252}
 253
 254/**
 255 * cgroup_is_descendant - test ancestry
 256 * @cgrp: the cgroup to be tested
 257 * @ancestor: possible ancestor of @cgrp
 258 *
 259 * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
 260 * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
 261 * and @ancestor are accessible.
 262 */
 263bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
 264{
 265        while (cgrp) {
 266                if (cgrp == ancestor)
 267                        return true;
 268                cgrp = cgrp->parent;
 269        }
 270        return false;
 271}
 272EXPORT_SYMBOL_GPL(cgroup_is_descendant);
 273
 274static int cgroup_is_releasable(const struct cgroup *cgrp)
 275{
 276        const int bits =
 277                (1 << CGRP_RELEASABLE) |
 278                (1 << CGRP_NOTIFY_ON_RELEASE);
 279        return (cgrp->flags & bits) == bits;
 280}
 281
 282static int notify_on_release(const struct cgroup *cgrp)
 283{
 284        return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
 285}
 286
 287/**
 288 * for_each_subsys - iterate all loaded cgroup subsystems
 289 * @ss: the iteration cursor
 290 * @i: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
 291 *
 292 * Should be called under cgroup_mutex.
 293 */
 294#define for_each_subsys(ss, i)                                          \
 295        for ((i) = 0; (i) < CGROUP_SUBSYS_COUNT; (i)++)                 \
 296                if (({ lockdep_assert_held(&cgroup_mutex);              \
 297                       !((ss) = cgroup_subsys[i]); })) { }              \
 298                else
 299
 300/**
 301 * for_each_builtin_subsys - iterate all built-in cgroup subsystems
 302 * @ss: the iteration cursor
 303 * @i: the index of @ss, CGROUP_BUILTIN_SUBSYS_COUNT after reaching the end
 304 *
 305 * Bulit-in subsystems are always present and iteration itself doesn't
 306 * require any synchronization.
 307 */
 308#define for_each_builtin_subsys(ss, i)                                  \
 309        for ((i) = 0; (i) < CGROUP_BUILTIN_SUBSYS_COUNT &&              \
 310             (((ss) = cgroup_subsys[i]) || true); (i)++)
 311
 312/* iterate each subsystem attached to a hierarchy */
 313#define for_each_root_subsys(root, ss)                                  \
 314        list_for_each_entry((ss), &(root)->subsys_list, sibling)
 315
 316/* iterate across the active hierarchies */
 317#define for_each_active_root(root)                                      \
 318        list_for_each_entry((root), &cgroup_roots, root_list)
 319
 320static inline struct cgroup *__d_cgrp(struct dentry *dentry)
 321{
 322        return dentry->d_fsdata;
 323}
 324
 325static inline struct cfent *__d_cfe(struct dentry *dentry)
 326{
 327        return dentry->d_fsdata;
 328}
 329
 330static inline struct cftype *__d_cft(struct dentry *dentry)
 331{
 332        return __d_cfe(dentry)->type;
 333}
 334
 335/**
 336 * cgroup_lock_live_group - take cgroup_mutex and check that cgrp is alive.
 337 * @cgrp: the cgroup to be checked for liveness
 338 *
 339 * On success, returns true; the mutex should be later unlocked.  On
 340 * failure returns false with no lock held.
 341 */
 342static bool cgroup_lock_live_group(struct cgroup *cgrp)
 343{
 344        mutex_lock(&cgroup_mutex);
 345        if (cgroup_is_dead(cgrp)) {
 346                mutex_unlock(&cgroup_mutex);
 347                return false;
 348        }
 349        return true;
 350}
 351
 352/* the list of cgroups eligible for automatic release. Protected by
 353 * release_list_lock */
 354static LIST_HEAD(release_list);
 355static DEFINE_RAW_SPINLOCK(release_list_lock);
 356static void cgroup_release_agent(struct work_struct *work);
 357static DECLARE_WORK(release_agent_work, cgroup_release_agent);
 358static void check_for_release(struct cgroup *cgrp);
 359
 360/*
 361 * A cgroup can be associated with multiple css_sets as different tasks may
 362 * belong to different cgroups on different hierarchies.  In the other
 363 * direction, a css_set is naturally associated with multiple cgroups.
 364 * This M:N relationship is represented by the following link structure
 365 * which exists for each association and allows traversing the associations
 366 * from both sides.
 367 */
 368struct cgrp_cset_link {
 369        /* the cgroup and css_set this link associates */
 370        struct cgroup           *cgrp;
 371        struct css_set          *cset;
 372
 373        /* list of cgrp_cset_links anchored at cgrp->cset_links */
 374        struct list_head        cset_link;
 375
 376        /* list of cgrp_cset_links anchored at css_set->cgrp_links */
 377        struct list_head        cgrp_link;
 378};
 379
 380/* The default css_set - used by init and its children prior to any
 381 * hierarchies being mounted. It contains a pointer to the root state
 382 * for each subsystem. Also used to anchor the list of css_sets. Not
 383 * reference-counted, to improve performance when child cgroups
 384 * haven't been created.
 385 */
 386
 387static struct css_set init_css_set;
 388static struct cgrp_cset_link init_cgrp_cset_link;
 389
 390static int cgroup_init_idr(struct cgroup_subsys *ss,
 391                           struct cgroup_subsys_state *css);
 392
 393/*
 394 * css_set_lock protects the list of css_set objects, and the chain of
 395 * tasks off each css_set.  Nests outside task->alloc_lock due to
 396 * css_task_iter_start().
 397 */
 398static DEFINE_RWLOCK(css_set_lock);
 399static int css_set_count;
 400
 401/*
 402 * hash table for cgroup groups. This improves the performance to find
 403 * an existing css_set. This hash doesn't (currently) take into
 404 * account cgroups in empty hierarchies.
 405 */
 406#define CSS_SET_HASH_BITS       7
 407static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
 408
 409static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
 410{
 411        unsigned long key = 0UL;
 412        struct cgroup_subsys *ss;
 413        int i;
 414
 415        for_each_subsys(ss, i)
 416                key += (unsigned long)css[i];
 417        key = (key >> 16) ^ key;
 418
 419        return key;
 420}
 421
 422/*
 423 * We don't maintain the lists running through each css_set to its task
 424 * until after the first call to css_task_iter_start().  This reduces the
 425 * fork()/exit() overhead for people who have cgroups compiled into their
 426 * kernel but not actually in use.
 427 */
 428static int use_task_css_set_links __read_mostly;
 429
 430static void __put_css_set(struct css_set *cset, int taskexit)
 431{
 432        struct cgrp_cset_link *link, *tmp_link;
 433
 434        /*
 435         * Ensure that the refcount doesn't hit zero while any readers
 436         * can see it. Similar to atomic_dec_and_lock(), but for an
 437         * rwlock
 438         */
 439        if (atomic_add_unless(&cset->refcount, -1, 1))
 440                return;
 441        write_lock(&css_set_lock);
 442        if (!atomic_dec_and_test(&cset->refcount)) {
 443                write_unlock(&css_set_lock);
 444                return;
 445        }
 446
 447        /* This css_set is dead. unlink it and release cgroup refcounts */
 448        hash_del(&cset->hlist);
 449        css_set_count--;
 450
 451        list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
 452                struct cgroup *cgrp = link->cgrp;
 453
 454                list_del(&link->cset_link);
 455                list_del(&link->cgrp_link);
 456
 457                /* @cgrp can't go away while we're holding css_set_lock */
 458                if (list_empty(&cgrp->cset_links) && notify_on_release(cgrp)) {
 459                        if (taskexit)
 460                                set_bit(CGRP_RELEASABLE, &cgrp->flags);
 461                        check_for_release(cgrp);
 462                }
 463
 464                kfree(link);
 465        }
 466
 467        write_unlock(&css_set_lock);
 468        kfree_rcu(cset, rcu_head);
 469}
 470
 471/*
 472 * refcounted get/put for css_set objects
 473 */
 474static inline void get_css_set(struct css_set *cset)
 475{
 476        atomic_inc(&cset->refcount);
 477}
 478
 479static inline void put_css_set(struct css_set *cset)
 480{
 481        __put_css_set(cset, 0);
 482}
 483
 484static inline void put_css_set_taskexit(struct css_set *cset)
 485{
 486        __put_css_set(cset, 1);
 487}
 488
 489/**
 490 * compare_css_sets - helper function for find_existing_css_set().
 491 * @cset: candidate css_set being tested
 492 * @old_cset: existing css_set for a task
 493 * @new_cgrp: cgroup that's being entered by the task
 494 * @template: desired set of css pointers in css_set (pre-calculated)
 495 *
 496 * Returns true if "cset" matches "old_cset" except for the hierarchy
 497 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 498 */
 499static bool compare_css_sets(struct css_set *cset,
 500                             struct css_set *old_cset,
 501                             struct cgroup *new_cgrp,
 502                             struct cgroup_subsys_state *template[])
 503{
 504        struct list_head *l1, *l2;
 505
 506        if (memcmp(template, cset->subsys, sizeof(cset->subsys))) {
 507                /* Not all subsystems matched */
 508                return false;
 509        }
 510
 511        /*
 512         * Compare cgroup pointers in order to distinguish between
 513         * different cgroups in heirarchies with no subsystems. We
 514         * could get by with just this check alone (and skip the
 515         * memcmp above) but on most setups the memcmp check will
 516         * avoid the need for this more expensive check on almost all
 517         * candidates.
 518         */
 519
 520        l1 = &cset->cgrp_links;
 521        l2 = &old_cset->cgrp_links;
 522        while (1) {
 523                struct cgrp_cset_link *link1, *link2;
 524                struct cgroup *cgrp1, *cgrp2;
 525
 526                l1 = l1->next;
 527                l2 = l2->next;
 528                /* See if we reached the end - both lists are equal length. */
 529                if (l1 == &cset->cgrp_links) {
 530                        BUG_ON(l2 != &old_cset->cgrp_links);
 531                        break;
 532                } else {
 533                        BUG_ON(l2 == &old_cset->cgrp_links);
 534                }
 535                /* Locate the cgroups associated with these links. */
 536                link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
 537                link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
 538                cgrp1 = link1->cgrp;
 539                cgrp2 = link2->cgrp;
 540                /* Hierarchies should be linked in the same order. */
 541                BUG_ON(cgrp1->root != cgrp2->root);
 542
 543                /*
 544                 * If this hierarchy is the hierarchy of the cgroup
 545                 * that's changing, then we need to check that this
 546                 * css_set points to the new cgroup; if it's any other
 547                 * hierarchy, then this css_set should point to the
 548                 * same cgroup as the old css_set.
 549                 */
 550                if (cgrp1->root == new_cgrp->root) {
 551                        if (cgrp1 != new_cgrp)
 552                                return false;
 553                } else {
 554                        if (cgrp1 != cgrp2)
 555                                return false;
 556                }
 557        }
 558        return true;
 559}
 560
 561/**
 562 * find_existing_css_set - init css array and find the matching css_set
 563 * @old_cset: the css_set that we're using before the cgroup transition
 564 * @cgrp: the cgroup that we're moving into
 565 * @template: out param for the new set of csses, should be clear on entry
 566 */
 567static struct css_set *find_existing_css_set(struct css_set *old_cset,
 568                                        struct cgroup *cgrp,
 569                                        struct cgroup_subsys_state *template[])
 570{
 571        struct cgroupfs_root *root = cgrp->root;
 572        struct cgroup_subsys *ss;
 573        struct css_set *cset;
 574        unsigned long key;
 575        int i;
 576
 577        /*
 578         * Build the set of subsystem state objects that we want to see in the
 579         * new css_set. while subsystems can change globally, the entries here
 580         * won't change, so no need for locking.
 581         */
 582        for_each_subsys(ss, i) {
 583                if (root->subsys_mask & (1UL << i)) {
 584                        /* Subsystem is in this hierarchy. So we want
 585                         * the subsystem state from the new
 586                         * cgroup */
 587                        template[i] = cgroup_css(cgrp, ss);
 588                } else {
 589                        /* Subsystem is not in this hierarchy, so we
 590                         * don't want to change the subsystem state */
 591                        template[i] = old_cset->subsys[i];
 592                }
 593        }
 594
 595        key = css_set_hash(template);
 596        hash_for_each_possible(css_set_table, cset, hlist, key) {
 597                if (!compare_css_sets(cset, old_cset, cgrp, template))
 598                        continue;
 599
 600                /* This css_set matches what we need */
 601                return cset;
 602        }
 603
 604        /* No existing cgroup group matched */
 605        return NULL;
 606}
 607
 608static void free_cgrp_cset_links(struct list_head *links_to_free)
 609{
 610        struct cgrp_cset_link *link, *tmp_link;
 611
 612        list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
 613                list_del(&link->cset_link);
 614                kfree(link);
 615        }
 616}
 617
 618/**
 619 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 620 * @count: the number of links to allocate
 621 * @tmp_links: list_head the allocated links are put on
 622 *
 623 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 624 * through ->cset_link.  Returns 0 on success or -errno.
 625 */
 626static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
 627{
 628        struct cgrp_cset_link *link;
 629        int i;
 630
 631        INIT_LIST_HEAD(tmp_links);
 632
 633        for (i = 0; i < count; i++) {
 634                link = kzalloc(sizeof(*link), GFP_KERNEL);
 635                if (!link) {
 636                        free_cgrp_cset_links(tmp_links);
 637                        return -ENOMEM;
 638                }
 639                list_add(&link->cset_link, tmp_links);
 640        }
 641        return 0;
 642}
 643
 644/**
 645 * link_css_set - a helper function to link a css_set to a cgroup
 646 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
 647 * @cset: the css_set to be linked
 648 * @cgrp: the destination cgroup
 649 */
 650static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
 651                         struct cgroup *cgrp)
 652{
 653        struct cgrp_cset_link *link;
 654
 655        BUG_ON(list_empty(tmp_links));
 656        link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
 657        link->cset = cset;
 658        link->cgrp = cgrp;
 659        list_move(&link->cset_link, &cgrp->cset_links);
 660        /*
 661         * Always add links to the tail of the list so that the list
 662         * is sorted by order of hierarchy creation
 663         */
 664        list_add_tail(&link->cgrp_link, &cset->cgrp_links);
 665}
 666
 667/**
 668 * find_css_set - return a new css_set with one cgroup updated
 669 * @old_cset: the baseline css_set
 670 * @cgrp: the cgroup to be updated
 671 *
 672 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 673 * substituted into the appropriate hierarchy.
 674 */
 675static struct css_set *find_css_set(struct css_set *old_cset,
 676                                    struct cgroup *cgrp)
 677{
 678        struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
 679        struct css_set *cset;
 680        struct list_head tmp_links;
 681        struct cgrp_cset_link *link;
 682        unsigned long key;
 683
 684        lockdep_assert_held(&cgroup_mutex);
 685
 686        /* First see if we already have a cgroup group that matches
 687         * the desired set */
 688        read_lock(&css_set_lock);
 689        cset = find_existing_css_set(old_cset, cgrp, template);
 690        if (cset)
 691                get_css_set(cset);
 692        read_unlock(&css_set_lock);
 693
 694        if (cset)
 695                return cset;
 696
 697        cset = kzalloc(sizeof(*cset), GFP_KERNEL);
 698        if (!cset)
 699                return NULL;
 700
 701        /* Allocate all the cgrp_cset_link objects that we'll need */
 702        if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
 703                kfree(cset);
 704                return NULL;
 705        }
 706
 707        atomic_set(&cset->refcount, 1);
 708        INIT_LIST_HEAD(&cset->cgrp_links);
 709        INIT_LIST_HEAD(&cset->tasks);
 710        INIT_HLIST_NODE(&cset->hlist);
 711
 712        /* Copy the set of subsystem state objects generated in
 713         * find_existing_css_set() */
 714        memcpy(cset->subsys, template, sizeof(cset->subsys));
 715
 716        write_lock(&css_set_lock);
 717        /* Add reference counts and links from the new css_set. */
 718        list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
 719                struct cgroup *c = link->cgrp;
 720
 721                if (c->root == cgrp->root)
 722                        c = cgrp;
 723                link_css_set(&tmp_links, cset, c);
 724        }
 725
 726        BUG_ON(!list_empty(&tmp_links));
 727
 728        css_set_count++;
 729
 730        /* Add this cgroup group to the hash table */
 731        key = css_set_hash(cset->subsys);
 732        hash_add(css_set_table, &cset->hlist, key);
 733
 734        write_unlock(&css_set_lock);
 735
 736        return cset;
 737}
 738
 739/*
 740 * Return the cgroup for "task" from the given hierarchy. Must be
 741 * called with cgroup_mutex held.
 742 */
 743static struct cgroup *task_cgroup_from_root(struct task_struct *task,
 744                                            struct cgroupfs_root *root)
 745{
 746        struct css_set *cset;
 747        struct cgroup *res = NULL;
 748
 749        BUG_ON(!mutex_is_locked(&cgroup_mutex));
 750        read_lock(&css_set_lock);
 751        /*
 752         * No need to lock the task - since we hold cgroup_mutex the
 753         * task can't change groups, so the only thing that can happen
 754         * is that it exits and its css is set back to init_css_set.
 755         */
 756        cset = task_css_set(task);
 757        if (cset == &init_css_set) {
 758                res = &root->top_cgroup;
 759        } else {
 760                struct cgrp_cset_link *link;
 761
 762                list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
 763                        struct cgroup *c = link->cgrp;
 764
 765                        if (c->root == root) {
 766                                res = c;
 767                                break;
 768                        }
 769                }
 770        }
 771        read_unlock(&css_set_lock);
 772        BUG_ON(!res);
 773        return res;
 774}
 775
 776/*
 777 * There is one global cgroup mutex. We also require taking
 778 * task_lock() when dereferencing a task's cgroup subsys pointers.
 779 * See "The task_lock() exception", at the end of this comment.
 780 *
 781 * A task must hold cgroup_mutex to modify cgroups.
 782 *
 783 * Any task can increment and decrement the count field without lock.
 784 * So in general, code holding cgroup_mutex can't rely on the count
 785 * field not changing.  However, if the count goes to zero, then only
 786 * cgroup_attach_task() can increment it again.  Because a count of zero
 787 * means that no tasks are currently attached, therefore there is no
 788 * way a task attached to that cgroup can fork (the other way to
 789 * increment the count).  So code holding cgroup_mutex can safely
 790 * assume that if the count is zero, it will stay zero. Similarly, if
 791 * a task holds cgroup_mutex on a cgroup with zero count, it
 792 * knows that the cgroup won't be removed, as cgroup_rmdir()
 793 * needs that mutex.
 794 *
 795 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
 796 * (usually) take cgroup_mutex.  These are the two most performance
 797 * critical pieces of code here.  The exception occurs on cgroup_exit(),
 798 * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
 799 * is taken, and if the cgroup count is zero, a usermode call made
 800 * to the release agent with the name of the cgroup (path relative to
 801 * the root of cgroup file system) as the argument.
 802 *
 803 * A cgroup can only be deleted if both its 'count' of using tasks
 804 * is zero, and its list of 'children' cgroups is empty.  Since all
 805 * tasks in the system use _some_ cgroup, and since there is always at
 806 * least one task in the system (init, pid == 1), therefore, top_cgroup
 807 * always has either children cgroups and/or using tasks.  So we don't
 808 * need a special hack to ensure that top_cgroup cannot be deleted.
 809 *
 810 *      The task_lock() exception
 811 *
 812 * The need for this exception arises from the action of
 813 * cgroup_attach_task(), which overwrites one task's cgroup pointer with
 814 * another.  It does so using cgroup_mutex, however there are
 815 * several performance critical places that need to reference
 816 * task->cgroup without the expense of grabbing a system global
 817 * mutex.  Therefore except as noted below, when dereferencing or, as
 818 * in cgroup_attach_task(), modifying a task's cgroup pointer we use
 819 * task_lock(), which acts on a spinlock (task->alloc_lock) already in
 820 * the task_struct routinely used for such matters.
 821 *
 822 * P.S.  One more locking exception.  RCU is used to guard the
 823 * update of a tasks cgroup pointer by cgroup_attach_task()
 824 */
 825
 826/*
 827 * A couple of forward declarations required, due to cyclic reference loop:
 828 * cgroup_mkdir -> cgroup_create -> cgroup_populate_dir ->
 829 * cgroup_add_file -> cgroup_create_file -> cgroup_dir_inode_operations
 830 * -> cgroup_mkdir.
 831 */
 832
 833static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
 834static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry);
 835static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask);
 836static const struct inode_operations cgroup_dir_inode_operations;
 837static const struct file_operations proc_cgroupstats_operations;
 838
 839static struct backing_dev_info cgroup_backing_dev_info = {
 840        .name           = "cgroup",
 841        .capabilities   = BDI_CAP_NO_ACCT_AND_WRITEBACK,
 842};
 843
 844static int alloc_css_id(struct cgroup_subsys_state *child_css);
 845
 846static struct inode *cgroup_new_inode(umode_t mode, struct super_block *sb)
 847{
 848        struct inode *inode = new_inode(sb);
 849
 850        if (inode) {
 851                inode->i_ino = get_next_ino();
 852                inode->i_mode = mode;
 853                inode->i_uid = current_fsuid();
 854                inode->i_gid = current_fsgid();
 855                inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 856                inode->i_mapping->backing_dev_info = &cgroup_backing_dev_info;
 857        }
 858        return inode;
 859}
 860
 861static struct cgroup_name *cgroup_alloc_name(struct dentry *dentry)
 862{
 863        struct cgroup_name *name;
 864
 865        name = kmalloc(sizeof(*name) + dentry->d_name.len + 1, GFP_KERNEL);
 866        if (!name)
 867                return NULL;
 868        strcpy(name->name, dentry->d_name.name);
 869        return name;
 870}
 871
 872static void cgroup_free_fn(struct work_struct *work)
 873{
 874        struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
 875
 876        mutex_lock(&cgroup_mutex);
 877        cgrp->root->number_of_cgroups--;
 878        mutex_unlock(&cgroup_mutex);
 879
 880        /*
 881         * We get a ref to the parent's dentry, and put the ref when
 882         * this cgroup is being freed, so it's guaranteed that the
 883         * parent won't be destroyed before its children.
 884         */
 885        dput(cgrp->parent->dentry);
 886
 887        /*
 888         * Drop the active superblock reference that we took when we
 889         * created the cgroup. This will free cgrp->root, if we are
 890         * holding the last reference to @sb.
 891         */
 892        deactivate_super(cgrp->root->sb);
 893
 894        /*
 895         * if we're getting rid of the cgroup, refcount should ensure
 896         * that there are no pidlists left.
 897         */
 898        BUG_ON(!list_empty(&cgrp->pidlists));
 899
 900        simple_xattrs_free(&cgrp->xattrs);
 901
 902        kfree(rcu_dereference_raw(cgrp->name));
 903        kfree(cgrp);
 904}
 905
 906static void cgroup_free_rcu(struct rcu_head *head)
 907{
 908        struct cgroup *cgrp = container_of(head, struct cgroup, rcu_head);
 909
 910        INIT_WORK(&cgrp->destroy_work, cgroup_free_fn);
 911        schedule_work(&cgrp->destroy_work);
 912}
 913
 914static void cgroup_diput(struct dentry *dentry, struct inode *inode)
 915{
 916        /* is dentry a directory ? if so, kfree() associated cgroup */
 917        if (S_ISDIR(inode->i_mode)) {
 918                struct cgroup *cgrp = dentry->d_fsdata;
 919
 920                BUG_ON(!(cgroup_is_dead(cgrp)));
 921                call_rcu(&cgrp->rcu_head, cgroup_free_rcu);
 922        } else {
 923                struct cfent *cfe = __d_cfe(dentry);
 924                struct cgroup *cgrp = dentry->d_parent->d_fsdata;
 925
 926                WARN_ONCE(!list_empty(&cfe->node) &&
 927                          cgrp != &cgrp->root->top_cgroup,
 928                          "cfe still linked for %s\n", cfe->type->name);
 929                simple_xattrs_free(&cfe->xattrs);
 930                kfree(cfe);
 931        }
 932        iput(inode);
 933}
 934
 935static int cgroup_delete(const struct dentry *d)
 936{
 937        return 1;
 938}
 939
 940static void remove_dir(struct dentry *d)
 941{
 942        struct dentry *parent = dget(d->d_parent);
 943
 944        d_delete(d);
 945        simple_rmdir(parent->d_inode, d);
 946        dput(parent);
 947}
 948
 949static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
 950{
 951        struct cfent *cfe;
 952
 953        lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
 954        lockdep_assert_held(&cgroup_mutex);
 955
 956        /*
 957         * If we're doing cleanup due to failure of cgroup_create(),
 958         * the corresponding @cfe may not exist.
 959         */
 960        list_for_each_entry(cfe, &cgrp->files, node) {
 961                struct dentry *d = cfe->dentry;
 962
 963                if (cft && cfe->type != cft)
 964                        continue;
 965
 966                dget(d);
 967                d_delete(d);
 968                simple_unlink(cgrp->dentry->d_inode, d);
 969                list_del_init(&cfe->node);
 970                dput(d);
 971
 972                break;
 973        }
 974}
 975
 976/**
 977 * cgroup_clear_dir - remove subsys files in a cgroup directory
 978 * @cgrp: target cgroup
 979 * @subsys_mask: mask of the subsystem ids whose files should be removed
 980 */
 981static void cgroup_clear_dir(struct cgroup *cgrp, unsigned long subsys_mask)
 982{
 983        struct cgroup_subsys *ss;
 984        int i;
 985
 986        for_each_subsys(ss, i) {
 987                struct cftype_set *set;
 988
 989                if (!test_bit(i, &subsys_mask))
 990                        continue;
 991                list_for_each_entry(set, &ss->cftsets, node)
 992                        cgroup_addrm_files(cgrp, set->cfts, false);
 993        }
 994}
 995
 996/*
 997 * NOTE : the dentry must have been dget()'ed
 998 */
 999static void cgroup_d_remove_dir(struct dentry *dentry)
1000{
1001        struct dentry *parent;
1002
1003        parent = dentry->d_parent;
1004        spin_lock(&parent->d_lock);
1005        spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1006        list_del_init(&dentry->d_u.d_child);
1007        spin_unlock(&dentry->d_lock);
1008        spin_unlock(&parent->d_lock);
1009        remove_dir(dentry);
1010}
1011
1012/*
1013 * Call with cgroup_mutex held. Drops reference counts on modules, including
1014 * any duplicate ones that parse_cgroupfs_options took. If this function
1015 * returns an error, no reference counts are touched.
1016 */
1017static int rebind_subsystems(struct cgroupfs_root *root,
1018                             unsigned long added_mask, unsigned removed_mask)
1019{
1020        struct cgroup *cgrp = &root->top_cgroup;
1021        struct cgroup_subsys *ss;
1022        unsigned long pinned = 0;
1023        int i, ret;
1024
1025        BUG_ON(!mutex_is_locked(&cgroup_mutex));
1026        BUG_ON(!mutex_is_locked(&cgroup_root_mutex));
1027
1028        /* Check that any added subsystems are currently free */
1029        for_each_subsys(ss, i) {
1030                if (!(added_mask & (1 << i)))
1031                        continue;
1032
1033                /* is the subsystem mounted elsewhere? */
1034                if (ss->root != &cgroup_dummy_root) {
1035                        ret = -EBUSY;
1036                        goto out_put;
1037                }
1038
1039                /* pin the module */
1040                if (!try_module_get(ss->module)) {
1041                        ret = -ENOENT;
1042                        goto out_put;
1043                }
1044                pinned |= 1 << i;
1045        }
1046
1047        /* subsys could be missing if unloaded between parsing and here */
1048        if (added_mask != pinned) {
1049                ret = -ENOENT;
1050                goto out_put;
1051        }
1052
1053        ret = cgroup_populate_dir(cgrp, added_mask);
1054        if (ret)
1055                goto out_put;
1056
1057        /*
1058         * Nothing can fail from this point on.  Remove files for the
1059         * removed subsystems and rebind each subsystem.
1060         */
1061        cgroup_clear_dir(cgrp, removed_mask);
1062
1063        for_each_subsys(ss, i) {
1064                unsigned long bit = 1UL << i;
1065
1066                if (bit & added_mask) {
1067                        /* We're binding this subsystem to this hierarchy */
1068                        BUG_ON(cgroup_css(cgrp, ss));
1069                        BUG_ON(!cgroup_css(cgroup_dummy_top, ss));
1070                        BUG_ON(cgroup_css(cgroup_dummy_top, ss)->cgroup != cgroup_dummy_top);
1071
1072                        rcu_assign_pointer(cgrp->subsys[i],
1073                                           cgroup_css(cgroup_dummy_top, ss));
1074                        cgroup_css(cgrp, ss)->cgroup = cgrp;
1075
1076                        list_move(&ss->sibling, &root->subsys_list);
1077                        ss->root = root;
1078                        if (ss->bind)
1079                                ss->bind(cgroup_css(cgrp, ss));
1080
1081                        /* refcount was already taken, and we're keeping it */
1082                        root->subsys_mask |= bit;
1083                } else if (bit & removed_mask) {
1084                        /* We're removing this subsystem */
1085                        BUG_ON(cgroup_css(cgrp, ss) != cgroup_css(cgroup_dummy_top, ss));
1086                        BUG_ON(cgroup_css(cgrp, ss)->cgroup != cgrp);
1087
1088                        if (ss->bind)
1089                                ss->bind(cgroup_css(cgroup_dummy_top, ss));
1090
1091                        cgroup_css(cgroup_dummy_top, ss)->cgroup = cgroup_dummy_top;
1092                        RCU_INIT_POINTER(cgrp->subsys[i], NULL);
1093
1094                        cgroup_subsys[i]->root = &cgroup_dummy_root;
1095                        list_move(&ss->sibling, &cgroup_dummy_root.subsys_list);
1096
1097                        /* subsystem is now free - drop reference on module */
1098                        module_put(ss->module);
1099                        root->subsys_mask &= ~bit;
1100                }
1101        }
1102
1103        /*
1104         * Mark @root has finished binding subsystems.  @root->subsys_mask
1105         * now matches the bound subsystems.
1106         */
1107        root->flags |= CGRP_ROOT_SUBSYS_BOUND;
1108
1109        return 0;
1110
1111out_put:
1112        for_each_subsys(ss, i)
1113                if (pinned & (1 << i))
1114                        module_put(ss->module);
1115        return ret;
1116}
1117
1118static int cgroup_show_options(struct seq_file *seq, struct dentry *dentry)
1119{
1120        struct cgroupfs_root *root = dentry->d_sb->s_fs_info;
1121        struct cgroup_subsys *ss;
1122
1123        mutex_lock(&cgroup_root_mutex);
1124        for_each_root_subsys(root, ss)
1125                seq_printf(seq, ",%s", ss->name);
1126        if (root->flags & CGRP_ROOT_SANE_BEHAVIOR)
1127                seq_puts(seq, ",sane_behavior");
1128        if (root->flags & CGRP_ROOT_NOPREFIX)
1129                seq_puts(seq, ",noprefix");
1130        if (root->flags & CGRP_ROOT_XATTR)
1131                seq_puts(seq, ",xattr");
1132        if (strlen(root->release_agent_path))
1133                seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1134        if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->top_cgroup.flags))
1135                seq_puts(seq, ",clone_children");
1136        if (strlen(root->name))
1137                seq_printf(seq, ",name=%s", root->name);
1138        mutex_unlock(&cgroup_root_mutex);
1139        return 0;
1140}
1141
1142struct cgroup_sb_opts {
1143        unsigned long subsys_mask;
1144        unsigned long flags;
1145        char *release_agent;
1146        bool cpuset_clone_children;
1147        char *name;
1148        /* User explicitly requested empty subsystem */
1149        bool none;
1150
1151        struct cgroupfs_root *new_root;
1152
1153};
1154
1155/*
1156 * Convert a hierarchy specifier into a bitmask of subsystems and
1157 * flags. Call with cgroup_mutex held to protect the cgroup_subsys[]
1158 * array. This function takes refcounts on subsystems to be used, unless it
1159 * returns error, in which case no refcounts are taken.
1160 */
1161static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1162{
1163        char *token, *o = data;
1164        bool all_ss = false, one_ss = false;
1165        unsigned long mask = (unsigned long)-1;
1166        struct cgroup_subsys *ss;
1167        int i;
1168
1169        BUG_ON(!mutex_is_locked(&cgroup_mutex));
1170
1171#ifdef CONFIG_CPUSETS
1172        mask = ~(1UL << cpuset_subsys_id);
1173#endif
1174
1175        memset(opts, 0, sizeof(*opts));
1176
1177        while ((token = strsep(&o, ",")) != NULL) {
1178                if (!*token)
1179                        return -EINVAL;
1180                if (!strcmp(token, "none")) {
1181                        /* Explicitly have no subsystems */
1182                        opts->none = true;
1183                        continue;
1184                }
1185                if (!strcmp(token, "all")) {
1186                        /* Mutually exclusive option 'all' + subsystem name */
1187                        if (one_ss)
1188                                return -EINVAL;
1189                        all_ss = true;
1190                        continue;
1191                }
1192                if (!strcmp(token, "__DEVEL__sane_behavior")) {
1193                        opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
1194                        continue;
1195                }
1196                if (!strcmp(token, "noprefix")) {
1197                        opts->flags |= CGRP_ROOT_NOPREFIX;
1198                        continue;
1199                }
1200                if (!strcmp(token, "clone_children")) {
1201                        opts->cpuset_clone_children = true;
1202                        continue;
1203                }
1204                if (!strcmp(token, "xattr")) {
1205                        opts->flags |= CGRP_ROOT_XATTR;
1206                        continue;
1207                }
1208                if (!strncmp(token, "release_agent=", 14)) {
1209                        /* Specifying two release agents is forbidden */
1210                        if (opts->release_agent)
1211                                return -EINVAL;
1212                        opts->release_agent =
1213                                kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1214                        if (!opts->release_agent)
1215                                return -ENOMEM;
1216                        continue;
1217                }
1218                if (!strncmp(token, "name=", 5)) {
1219                        const char *name = token + 5;
1220                        /* Can't specify an empty name */
1221                        if (!strlen(name))
1222                                return -EINVAL;
1223                        /* Must match [\w.-]+ */
1224                        for (i = 0; i < strlen(name); i++) {
1225                                char c = name[i];
1226                                if (isalnum(c))
1227                                        continue;
1228                                if ((c == '.') || (c == '-') || (c == '_'))
1229                                        continue;
1230                                return -EINVAL;
1231                        }
1232                        /* Specifying two names is forbidden */
1233                        if (opts->name)
1234                                return -EINVAL;
1235                        opts->name = kstrndup(name,
1236                                              MAX_CGROUP_ROOT_NAMELEN - 1,
1237                                              GFP_KERNEL);
1238                        if (!opts->name)
1239                                return -ENOMEM;
1240
1241                        continue;
1242                }
1243
1244                for_each_subsys(ss, i) {
1245                        if (strcmp(token, ss->name))
1246                                continue;
1247                        if (ss->disabled)
1248                                continue;
1249
1250                        /* Mutually exclusive option 'all' + subsystem name */
1251                        if (all_ss)
1252                                return -EINVAL;
1253                        set_bit(i, &opts->subsys_mask);
1254                        one_ss = true;
1255
1256                        break;
1257                }
1258                if (i == CGROUP_SUBSYS_COUNT)
1259                        return -ENOENT;
1260        }
1261
1262        /*
1263         * If the 'all' option was specified select all the subsystems,
1264         * otherwise if 'none', 'name=' and a subsystem name options
1265         * were not specified, let's default to 'all'
1266         */
1267        if (all_ss || (!one_ss && !opts->none && !opts->name))
1268                for_each_subsys(ss, i)
1269                        if (!ss->disabled)
1270                                set_bit(i, &opts->subsys_mask);
1271
1272        /* Consistency checks */
1273
1274        if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1275                pr_warning("cgroup: sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");
1276
1277                if (opts->flags & CGRP_ROOT_NOPREFIX) {
1278                        pr_err("cgroup: sane_behavior: noprefix is not allowed\n");
1279                        return -EINVAL;
1280                }
1281
1282                if (opts->cpuset_clone_children) {
1283                        pr_err("cgroup: sane_behavior: clone_children is not allowed\n");
1284                        return -EINVAL;
1285                }
1286        }
1287
1288        /*
1289         * Option noprefix was introduced just for backward compatibility
1290         * with the old cpuset, so we allow noprefix only if mounting just
1291         * the cpuset subsystem.
1292         */
1293        if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1294                return -EINVAL;
1295
1296
1297        /* Can't specify "none" and some subsystems */
1298        if (opts->subsys_mask && opts->none)
1299                return -EINVAL;
1300
1301        /*
1302         * We either have to specify by name or by subsystems. (So all
1303         * empty hierarchies must have a name).
1304         */
1305        if (!opts->subsys_mask && !opts->name)
1306                return -EINVAL;
1307
1308        return 0;
1309}
1310
1311static int cgroup_remount(struct super_block *sb, int *flags, char *data)
1312{
1313        int ret = 0;
1314        struct cgroupfs_root *root = sb->s_fs_info;
1315        struct cgroup *cgrp = &root->top_cgroup;
1316        struct cgroup_sb_opts opts;
1317        unsigned long added_mask, removed_mask;
1318
1319        if (root->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1320                pr_err("cgroup: sane_behavior: remount is not allowed\n");
1321                return -EINVAL;
1322        }
1323
1324        mutex_lock(&cgrp->dentry->d_inode->i_mutex);
1325        mutex_lock(&cgroup_mutex);
1326        mutex_lock(&cgroup_root_mutex);
1327
1328        /* See what subsystems are wanted */
1329        ret = parse_cgroupfs_options(data, &opts);
1330        if (ret)
1331                goto out_unlock;
1332
1333        if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1334                pr_warning("cgroup: option changes via remount are deprecated (pid=%d comm=%s)\n",
1335                           task_tgid_nr(current), current->comm);
1336
1337        added_mask = opts.subsys_mask & ~root->subsys_mask;
1338        removed_mask = root->subsys_mask & ~opts.subsys_mask;
1339
1340        /* Don't allow flags or name to change at remount */
1341        if (((opts.flags ^ root->flags) & CGRP_ROOT_OPTION_MASK) ||
1342            (opts.name && strcmp(opts.name, root->name))) {
1343                pr_err("cgroup: option or name mismatch, new: 0x%lx \"%s\", old: 0x%lx \"%s\"\n",
1344                       opts.flags & CGRP_ROOT_OPTION_MASK, opts.name ?: "",
1345                       root->flags & CGRP_ROOT_OPTION_MASK, root->name);
1346                ret = -EINVAL;
1347                goto out_unlock;
1348        }
1349
1350        /* remounting is not allowed for populated hierarchies */
1351        if (root->number_of_cgroups > 1) {
1352                ret = -EBUSY;
1353                goto out_unlock;
1354        }
1355
1356        ret = rebind_subsystems(root, added_mask, removed_mask);
1357        if (ret)
1358                goto out_unlock;
1359
1360        if (opts.release_agent)
1361                strcpy(root->release_agent_path, opts.release_agent);
1362 out_unlock:
1363        kfree(opts.release_agent);
1364        kfree(opts.name);
1365        mutex_unlock(&cgroup_root_mutex);
1366        mutex_unlock(&cgroup_mutex);
1367        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
1368        return ret;
1369}
1370
1371static const struct super_operations cgroup_ops = {
1372        .statfs = simple_statfs,
1373        .drop_inode = generic_delete_inode,
1374        .show_options = cgroup_show_options,
1375        .remount_fs = cgroup_remount,
1376};
1377
1378static void init_cgroup_housekeeping(struct cgroup *cgrp)
1379{
1380        INIT_LIST_HEAD(&cgrp->sibling);
1381        INIT_LIST_HEAD(&cgrp->children);
1382        INIT_LIST_HEAD(&cgrp->files);
1383        INIT_LIST_HEAD(&cgrp->cset_links);
1384        INIT_LIST_HEAD(&cgrp->release_list);
1385        INIT_LIST_HEAD(&cgrp->pidlists);
1386        mutex_init(&cgrp->pidlist_mutex);
1387        cgrp->dummy_css.cgroup = cgrp;
1388        INIT_LIST_HEAD(&cgrp->event_list);
1389        spin_lock_init(&cgrp->event_list_lock);
1390        simple_xattrs_init(&cgrp->xattrs);
1391}
1392
1393static void init_cgroup_root(struct cgroupfs_root *root)
1394{
1395        struct cgroup *cgrp = &root->top_cgroup;
1396
1397        INIT_LIST_HEAD(&root->subsys_list);
1398        INIT_LIST_HEAD(&root->root_list);
1399        root->number_of_cgroups = 1;
1400        cgrp->root = root;
1401        RCU_INIT_POINTER(cgrp->name, &root_cgroup_name);
1402        init_cgroup_housekeeping(cgrp);
1403        idr_init(&root->cgroup_idr);
1404}
1405
1406static int cgroup_init_root_id(struct cgroupfs_root *root, int start, int end)
1407{
1408        int id;
1409
1410        lockdep_assert_held(&cgroup_mutex);
1411        lockdep_assert_held(&cgroup_root_mutex);
1412
1413        id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, start, end,
1414                              GFP_KERNEL);
1415        if (id < 0)
1416                return id;
1417
1418        root->hierarchy_id = id;
1419        return 0;
1420}
1421
1422static void cgroup_exit_root_id(struct cgroupfs_root *root)
1423{
1424        lockdep_assert_held(&cgroup_mutex);
1425        lockdep_assert_held(&cgroup_root_mutex);
1426
1427        if (root->hierarchy_id) {
1428                idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1429                root->hierarchy_id = 0;
1430        }
1431}
1432
1433static int cgroup_test_super(struct super_block *sb, void *data)
1434{
1435        struct cgroup_sb_opts *opts = data;
1436        struct cgroupfs_root *root = sb->s_fs_info;
1437
1438        /* If we asked for a name then it must match */
1439        if (opts->name && strcmp(opts->name, root->name))
1440                return 0;
1441
1442        /*
1443         * If we asked for subsystems (or explicitly for no
1444         * subsystems) then they must match
1445         */
1446        if ((opts->subsys_mask || opts->none)
1447            && (opts->subsys_mask != root->subsys_mask))
1448                return 0;
1449
1450        return 1;
1451}
1452
1453static struct cgroupfs_root *cgroup_root_from_opts(struct cgroup_sb_opts *opts)
1454{
1455        struct cgroupfs_root *root;
1456
1457        if (!opts->subsys_mask && !opts->none)
1458                return NULL;
1459
1460        root = kzalloc(sizeof(*root), GFP_KERNEL);
1461        if (!root)
1462                return ERR_PTR(-ENOMEM);
1463
1464        init_cgroup_root(root);
1465
1466        /*
1467         * We need to set @root->subsys_mask now so that @root can be
1468         * matched by cgroup_test_super() before it finishes
1469         * initialization; otherwise, competing mounts with the same
1470         * options may try to bind the same subsystems instead of waiting
1471         * for the first one leading to unexpected mount errors.
1472         * SUBSYS_BOUND will be set once actual binding is complete.
1473         */
1474        root->subsys_mask = opts->subsys_mask;
1475        root->flags = opts->flags;
1476        if (opts->release_agent)
1477                strcpy(root->release_agent_path, opts->release_agent);
1478        if (opts->name)
1479                strcpy(root->name, opts->name);
1480        if (opts->cpuset_clone_children)
1481                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->top_cgroup.flags);
1482        return root;
1483}
1484
1485static void cgroup_free_root(struct cgroupfs_root *root)
1486{
1487        if (root) {
1488                /* hierarhcy ID shoulid already have been released */
1489                WARN_ON_ONCE(root->hierarchy_id);
1490
1491                idr_destroy(&root->cgroup_idr);
1492                kfree(root);
1493        }
1494}
1495
1496static int cgroup_set_super(struct super_block *sb, void *data)
1497{
1498        int ret;
1499        struct cgroup_sb_opts *opts = data;
1500
1501        /* If we don't have a new root, we can't set up a new sb */
1502        if (!opts->new_root)
1503                return -EINVAL;
1504
1505        BUG_ON(!opts->subsys_mask && !opts->none);
1506
1507        ret = set_anon_super(sb, NULL);
1508        if (ret)
1509                return ret;
1510
1511        sb->s_fs_info = opts->new_root;
1512        opts->new_root->sb = sb;
1513
1514        sb->s_blocksize = PAGE_CACHE_SIZE;
1515        sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
1516        sb->s_magic = CGROUP_SUPER_MAGIC;
1517        sb->s_op = &cgroup_ops;
1518
1519        return 0;
1520}
1521
1522static int cgroup_get_rootdir(struct super_block *sb)
1523{
1524        static const struct dentry_operations cgroup_dops = {
1525                .d_iput = cgroup_diput,
1526                .d_delete = cgroup_delete,
1527        };
1528
1529        struct inode *inode =
1530                cgroup_new_inode(S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR, sb);
1531
1532        if (!inode)
1533                return -ENOMEM;
1534
1535        inode->i_fop = &simple_dir_operations;
1536        inode->i_op = &cgroup_dir_inode_operations;
1537        /* directories start off with i_nlink == 2 (for "." entry) */
1538        inc_nlink(inode);
1539        sb->s_root = d_make_root(inode);
1540        if (!sb->s_root)
1541                return -ENOMEM;
1542        /* for everything else we want ->d_op set */
1543        sb->s_d_op = &cgroup_dops;
1544        return 0;
1545}
1546
1547static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1548                         int flags, const char *unused_dev_name,
1549                         void *data)
1550{
1551        struct cgroup_sb_opts opts;
1552        struct cgroupfs_root *root;
1553        int ret = 0;
1554        struct super_block *sb;
1555        struct cgroupfs_root *new_root;
1556        struct list_head tmp_links;
1557        struct inode *inode;
1558        const struct cred *cred;
1559
1560        /* First find the desired set of subsystems */
1561        mutex_lock(&cgroup_mutex);
1562        ret = parse_cgroupfs_options(data, &opts);
1563        mutex_unlock(&cgroup_mutex);
1564        if (ret)
1565                goto out_err;
1566
1567        /*
1568         * Allocate a new cgroup root. We may not need it if we're
1569         * reusing an existing hierarchy.
1570         */
1571        new_root = cgroup_root_from_opts(&opts);
1572        if (IS_ERR(new_root)) {
1573                ret = PTR_ERR(new_root);
1574                goto out_err;
1575        }
1576        opts.new_root = new_root;
1577
1578        /* Locate an existing or new sb for this hierarchy */
1579        sb = sget(fs_type, cgroup_test_super, cgroup_set_super, 0, &opts);
1580        if (IS_ERR(sb)) {
1581                ret = PTR_ERR(sb);
1582                cgroup_free_root(opts.new_root);
1583                goto out_err;
1584        }
1585
1586        root = sb->s_fs_info;
1587        BUG_ON(!root);
1588        if (root == opts.new_root) {
1589                /* We used the new root structure, so this is a new hierarchy */
1590                struct cgroup *root_cgrp = &root->top_cgroup;
1591                struct cgroupfs_root *existing_root;
1592                int i;
1593                struct css_set *cset;
1594
1595                BUG_ON(sb->s_root != NULL);
1596
1597                ret = cgroup_get_rootdir(sb);
1598                if (ret)
1599                        goto drop_new_super;
1600                inode = sb->s_root->d_inode;
1601
1602                mutex_lock(&inode->i_mutex);
1603                mutex_lock(&cgroup_mutex);
1604                mutex_lock(&cgroup_root_mutex);
1605
1606                root_cgrp->id = idr_alloc(&root->cgroup_idr, root_cgrp,
1607                                           0, 1, GFP_KERNEL);
1608                if (root_cgrp->id < 0)
1609                        goto unlock_drop;
1610
1611                /* Check for name clashes with existing mounts */
1612                ret = -EBUSY;
1613                if (strlen(root->name))
1614                        for_each_active_root(existing_root)
1615                                if (!strcmp(existing_root->name, root->name))
1616                                        goto unlock_drop;
1617
1618                /*
1619                 * We're accessing css_set_count without locking
1620                 * css_set_lock here, but that's OK - it can only be
1621                 * increased by someone holding cgroup_lock, and
1622                 * that's us. The worst that can happen is that we
1623                 * have some link structures left over
1624                 */
1625                ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
1626                if (ret)
1627                        goto unlock_drop;
1628
1629                /* ID 0 is reserved for dummy root, 1 for unified hierarchy */
1630                ret = cgroup_init_root_id(root, 2, 0);
1631                if (ret)
1632                        goto unlock_drop;
1633
1634                sb->s_root->d_fsdata = root_cgrp;
1635                root_cgrp->dentry = sb->s_root;
1636
1637                /*
1638                 * We're inside get_sb() and will call lookup_one_len() to
1639                 * create the root files, which doesn't work if SELinux is
1640                 * in use.  The following cred dancing somehow works around
1641                 * it.  See 2ce9738ba ("cgroupfs: use init_cred when
1642                 * populating new cgroupfs mount") for more details.
1643                 */
1644                cred = override_creds(&init_cred);
1645
1646                ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
1647                if (ret)
1648                        goto rm_base_files;
1649
1650                ret = rebind_subsystems(root, root->subsys_mask, 0);
1651                if (ret)
1652                        goto rm_base_files;
1653
1654                revert_creds(cred);
1655
1656                /*
1657                 * There must be no failure case after here, since rebinding
1658                 * takes care of subsystems' refcounts, which are explicitly
1659                 * dropped in the failure exit path.
1660                 */
1661
1662                list_add(&root->root_list, &cgroup_roots);
1663                cgroup_root_count++;
1664
1665                /* Link the top cgroup in this hierarchy into all
1666                 * the css_set objects */
1667                write_lock(&css_set_lock);
1668                hash_for_each(css_set_table, i, cset, hlist)
1669                        link_css_set(&tmp_links, cset, root_cgrp);
1670                write_unlock(&css_set_lock);
1671
1672                free_cgrp_cset_links(&tmp_links);
1673
1674                BUG_ON(!list_empty(&root_cgrp->children));
1675                BUG_ON(root->number_of_cgroups != 1);
1676
1677                mutex_unlock(&cgroup_root_mutex);
1678                mutex_unlock(&cgroup_mutex);
1679                mutex_unlock(&inode->i_mutex);
1680        } else {
1681                /*
1682                 * We re-used an existing hierarchy - the new root (if
1683                 * any) is not needed
1684                 */
1685                cgroup_free_root(opts.new_root);
1686
1687                if ((root->flags ^ opts.flags) & CGRP_ROOT_OPTION_MASK) {
1688                        if ((root->flags | opts.flags) & CGRP_ROOT_SANE_BEHAVIOR) {
1689                                pr_err("cgroup: sane_behavior: new mount options should match the existing superblock\n");
1690                                ret = -EINVAL;
1691                                goto drop_new_super;
1692                        } else {
1693                                pr_warning("cgroup: new mount options do not match the existing superblock, will be ignored\n");
1694                        }
1695                }
1696        }
1697
1698        kfree(opts.release_agent);
1699        kfree(opts.name);
1700        return dget(sb->s_root);
1701
1702 rm_base_files:
1703        free_cgrp_cset_links(&tmp_links);
1704        cgroup_addrm_files(&root->top_cgroup, cgroup_base_files, false);
1705        revert_creds(cred);
1706 unlock_drop:
1707        cgroup_exit_root_id(root);
1708        mutex_unlock(&cgroup_root_mutex);
1709        mutex_unlock(&cgroup_mutex);
1710        mutex_unlock(&inode->i_mutex);
1711 drop_new_super:
1712        deactivate_locked_super(sb);
1713 out_err:
1714        kfree(opts.release_agent);
1715        kfree(opts.name);
1716        return ERR_PTR(ret);
1717}
1718
1719static void cgroup_kill_sb(struct super_block *sb) {
1720        struct cgroupfs_root *root = sb->s_fs_info;
1721        struct cgroup *cgrp = &root->top_cgroup;
1722        struct cgrp_cset_link *link, *tmp_link;
1723        int ret;
1724
1725        BUG_ON(!root);
1726
1727        BUG_ON(root->number_of_cgroups != 1);
1728        BUG_ON(!list_empty(&cgrp->children));
1729
1730        mutex_lock(&cgrp->dentry->d_inode->i_mutex);
1731        mutex_lock(&cgroup_mutex);
1732        mutex_lock(&cgroup_root_mutex);
1733
1734        /* Rebind all subsystems back to the default hierarchy */
1735        if (root->flags & CGRP_ROOT_SUBSYS_BOUND) {
1736                ret = rebind_subsystems(root, 0, root->subsys_mask);
1737                /* Shouldn't be able to fail ... */
1738                BUG_ON(ret);
1739        }
1740
1741        /*
1742         * Release all the links from cset_links to this hierarchy's
1743         * root cgroup
1744         */
1745        write_lock(&css_set_lock);
1746
1747        list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
1748                list_del(&link->cset_link);
1749                list_del(&link->cgrp_link);
1750                kfree(link);
1751        }
1752        write_unlock(&css_set_lock);
1753
1754        if (!list_empty(&root->root_list)) {
1755                list_del(&root->root_list);
1756                cgroup_root_count--;
1757        }
1758
1759        cgroup_exit_root_id(root);
1760
1761        mutex_unlock(&cgroup_root_mutex);
1762        mutex_unlock(&cgroup_mutex);
1763        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
1764
1765        simple_xattrs_free(&cgrp->xattrs);
1766
1767        kill_litter_super(sb);
1768        cgroup_free_root(root);
1769}
1770
1771static struct file_system_type cgroup_fs_type = {
1772        .name = "cgroup",
1773        .mount = cgroup_mount,
1774        .kill_sb = cgroup_kill_sb,
1775};
1776
1777static struct kobject *cgroup_kobj;
1778
1779/**
1780 * cgroup_path - generate the path of a cgroup
1781 * @cgrp: the cgroup in question
1782 * @buf: the buffer to write the path into
1783 * @buflen: the length of the buffer
1784 *
1785 * Writes path of cgroup into buf.  Returns 0 on success, -errno on error.
1786 *
1787 * We can't generate cgroup path using dentry->d_name, as accessing
1788 * dentry->name must be protected by irq-unsafe dentry->d_lock or parent
1789 * inode's i_mutex, while on the other hand cgroup_path() can be called
1790 * with some irq-safe spinlocks held.
1791 */
1792int cgroup_path(const struct cgroup *cgrp, char *buf, int buflen)
1793{
1794        int ret = -ENAMETOOLONG;
1795        char *start;
1796
1797        if (!cgrp->parent) {
1798                if (strlcpy(buf, "/", buflen) >= buflen)
1799                        return -ENAMETOOLONG;
1800                return 0;
1801        }
1802
1803        start = buf + buflen - 1;
1804        *start = '\0';
1805
1806        rcu_read_lock();
1807        do {
1808                const char *name = cgroup_name(cgrp);
1809                int len;
1810
1811                len = strlen(name);
1812                if ((start -= len) < buf)
1813                        goto out;
1814                memcpy(start, name, len);
1815
1816                if (--start < buf)
1817                        goto out;
1818                *start = '/';
1819
1820                cgrp = cgrp->parent;
1821        } while (cgrp->parent);
1822        ret = 0;
1823        memmove(buf, start, buf + buflen - start);
1824out:
1825        rcu_read_unlock();
1826        return ret;
1827}
1828EXPORT_SYMBOL_GPL(cgroup_path);
1829
1830/**
1831 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1832 * @task: target task
1833 * @buf: the buffer to write the path into
1834 * @buflen: the length of the buffer
1835 *
1836 * Determine @task's cgroup on the first (the one with the lowest non-zero
1837 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
1838 * function grabs cgroup_mutex and shouldn't be used inside locks used by
1839 * cgroup controller callbacks.
1840 *
1841 * Returns 0 on success, fails with -%ENAMETOOLONG if @buflen is too short.
1842 */
1843int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1844{
1845        struct cgroupfs_root *root;
1846        struct cgroup *cgrp;
1847        int hierarchy_id = 1, ret = 0;
1848
1849        if (buflen < 2)
1850                return -ENAMETOOLONG;
1851
1852        mutex_lock(&cgroup_mutex);
1853
1854        root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
1855
1856        if (root) {
1857                cgrp = task_cgroup_from_root(task, root);
1858                ret = cgroup_path(cgrp, buf, buflen);
1859        } else {
1860                /* if no hierarchy exists, everyone is in "/" */
1861                memcpy(buf, "/", 2);
1862        }
1863
1864        mutex_unlock(&cgroup_mutex);
1865        return ret;
1866}
1867EXPORT_SYMBOL_GPL(task_cgroup_path);
1868
1869/*
1870 * Control Group taskset
1871 */
1872struct task_and_cgroup {
1873        struct task_struct      *task;
1874        struct cgroup           *cgrp;
1875        struct css_set          *cset;
1876};
1877
1878struct cgroup_taskset {
1879        struct task_and_cgroup  single;
1880        struct flex_array       *tc_array;
1881        int                     tc_array_len;
1882        int                     idx;
1883        struct cgroup           *cur_cgrp;
1884};
1885
1886/**
1887 * cgroup_taskset_first - reset taskset and return the first task
1888 * @tset: taskset of interest
1889 *
1890 * @tset iteration is initialized and the first task is returned.
1891 */
1892struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
1893{
1894        if (tset->tc_array) {
1895                tset->idx = 0;
1896                return cgroup_taskset_next(tset);
1897        } else {
1898                tset->cur_cgrp = tset->single.cgrp;
1899                return tset->single.task;
1900        }
1901}
1902EXPORT_SYMBOL_GPL(cgroup_taskset_first);
1903
1904/**
1905 * cgroup_taskset_next - iterate to the next task in taskset
1906 * @tset: taskset of interest
1907 *
1908 * Return the next task in @tset.  Iteration must have been initialized
1909 * with cgroup_taskset_first().
1910 */
1911struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
1912{
1913        struct task_and_cgroup *tc;
1914
1915        if (!tset->tc_array || tset->idx >= tset->tc_array_len)
1916                return NULL;
1917
1918        tc = flex_array_get(tset->tc_array, tset->idx++);
1919        tset->cur_cgrp = tc->cgrp;
1920        return tc->task;
1921}
1922EXPORT_SYMBOL_GPL(cgroup_taskset_next);
1923
1924/**
1925 * cgroup_taskset_cur_css - return the matching css for the current task
1926 * @tset: taskset of interest
1927 * @subsys_id: the ID of the target subsystem
1928 *
1929 * Return the css for the current (last returned) task of @tset for
1930 * subsystem specified by @subsys_id.  This function must be preceded by
1931 * either cgroup_taskset_first() or cgroup_taskset_next().
1932 */
1933struct cgroup_subsys_state *cgroup_taskset_cur_css(struct cgroup_taskset *tset,
1934                                                   int subsys_id)
1935{
1936        return cgroup_css(tset->cur_cgrp, cgroup_subsys[subsys_id]);
1937}
1938EXPORT_SYMBOL_GPL(cgroup_taskset_cur_css);
1939
1940/**
1941 * cgroup_taskset_size - return the number of tasks in taskset
1942 * @tset: taskset of interest
1943 */
1944int cgroup_taskset_size(struct cgroup_taskset *tset)
1945{
1946        return tset->tc_array ? tset->tc_array_len : 1;
1947}
1948EXPORT_SYMBOL_GPL(cgroup_taskset_size);
1949
1950
1951/*
1952 * cgroup_task_migrate - move a task from one cgroup to another.
1953 *
1954 * Must be called with cgroup_mutex and threadgroup locked.
1955 */
1956static void cgroup_task_migrate(struct cgroup *old_cgrp,
1957                                struct task_struct *tsk,
1958                                struct css_set *new_cset)
1959{
1960        struct css_set *old_cset;
1961
1962        /*
1963         * We are synchronized through threadgroup_lock() against PF_EXITING
1964         * setting such that we can't race against cgroup_exit() changing the
1965         * css_set to init_css_set and dropping the old one.
1966         */
1967        WARN_ON_ONCE(tsk->flags & PF_EXITING);
1968        old_cset = task_css_set(tsk);
1969
1970        task_lock(tsk);
1971        rcu_assign_pointer(tsk->cgroups, new_cset);
1972        task_unlock(tsk);
1973
1974        /* Update the css_set linked lists if we're using them */
1975        write_lock(&css_set_lock);
1976        if (!list_empty(&tsk->cg_list))
1977                list_move(&tsk->cg_list, &new_cset->tasks);
1978        write_unlock(&css_set_lock);
1979
1980        /*
1981         * We just gained a reference on old_cset by taking it from the
1982         * task. As trading it for new_cset is protected by cgroup_mutex,
1983         * we're safe to drop it here; it will be freed under RCU.
1984         */
1985        set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
1986        put_css_set(old_cset);
1987}
1988
1989/**
1990 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
1991 * @cgrp: the cgroup to attach to
1992 * @tsk: the task or the leader of the threadgroup to be attached
1993 * @threadgroup: attach the whole threadgroup?
1994 *
1995 * Call holding cgroup_mutex and the group_rwsem of the leader. Will take
1996 * task_lock of @tsk or each thread in the threadgroup individually in turn.
1997 */
1998static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
1999                              bool threadgroup)
2000{
2001        int retval, i, group_size;
2002        struct cgroup_subsys *ss, *failed_ss = NULL;
2003        struct cgroupfs_root *root = cgrp->root;
2004        /* threadgroup list cursor and array */
2005        struct task_struct *leader = tsk;
2006        struct task_and_cgroup *tc;
2007        struct flex_array *group;
2008        struct cgroup_taskset tset = { };
2009
2010        /*
2011         * step 0: in order to do expensive, possibly blocking operations for
2012         * every thread, we cannot iterate the thread group list, since it needs
2013         * rcu or tasklist locked. instead, build an array of all threads in the
2014         * group - group_rwsem prevents new threads from appearing, and if
2015         * threads exit, this will just be an over-estimate.
2016         */
2017        if (threadgroup)
2018                group_size = get_nr_threads(tsk);
2019        else
2020                group_size = 1;
2021        /* flex_array supports very large thread-groups better than kmalloc. */
2022        group = flex_array_alloc(sizeof(*tc), group_size, GFP_KERNEL);
2023        if (!group)
2024                return -ENOMEM;
2025        /* pre-allocate to guarantee space while iterating in rcu read-side. */
2026        retval = flex_array_prealloc(group, 0, group_size, GFP_KERNEL);
2027        if (retval)
2028                goto out_free_group_list;
2029
2030        i = 0;
2031        /*
2032         * Prevent freeing of tasks while we take a snapshot. Tasks that are
2033         * already PF_EXITING could be freed from underneath us unless we
2034         * take an rcu_read_lock.
2035         */
2036        rcu_read_lock();
2037        do {
2038                struct task_and_cgroup ent;
2039
2040                /* @tsk either already exited or can't exit until the end */
2041                if (tsk->flags & PF_EXITING)
2042                        goto next;
2043
2044                /* as per above, nr_threads may decrease, but not increase. */
2045                BUG_ON(i >= group_size);
2046                ent.task = tsk;
2047                ent.cgrp = task_cgroup_from_root(tsk, root);
2048                /* nothing to do if this task is already in the cgroup */
2049                if (ent.cgrp == cgrp)
2050                        goto next;
2051                /*
2052                 * saying GFP_ATOMIC has no effect here because we did prealloc
2053                 * earlier, but it's good form to communicate our expectations.
2054                 */
2055                retval = flex_array_put(group, i, &ent, GFP_ATOMIC);
2056                BUG_ON(retval != 0);
2057                i++;
2058        next:
2059                if (!threadgroup)
2060                        break;
2061        } while_each_thread(leader, tsk);
2062        rcu_read_unlock();
2063        /* remember the number of threads in the array for later. */
2064        group_size = i;
2065        tset.tc_array = group;
2066        tset.tc_array_len = group_size;
2067
2068        /* methods shouldn't be called if no task is actually migrating */
2069        retval = 0;
2070        if (!group_size)
2071                goto out_free_group_list;
2072
2073        /*
2074         * step 1: check that we can legitimately attach to the cgroup.
2075         */
2076        for_each_root_subsys(root, ss) {
2077                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
2078
2079                if (ss->can_attach) {
2080                        retval = ss->can_attach(css, &tset);
2081                        if (retval) {
2082                                failed_ss = ss;
2083                                goto out_cancel_attach;
2084                        }
2085                }
2086        }
2087
2088        /*
2089         * step 2: make sure css_sets exist for all threads to be migrated.
2090         * we use find_css_set, which allocates a new one if necessary.
2091         */
2092        for (i = 0; i < group_size; i++) {
2093                struct css_set *old_cset;
2094
2095                tc = flex_array_get(group, i);
2096                old_cset = task_css_set(tc->task);
2097                tc->cset = find_css_set(old_cset, cgrp);
2098                if (!tc->cset) {
2099                        retval = -ENOMEM;
2100                        goto out_put_css_set_refs;
2101                }
2102        }
2103
2104        /*
2105         * step 3: now that we're guaranteed success wrt the css_sets,
2106         * proceed to move all tasks to the new cgroup.  There are no
2107         * failure cases after here, so this is the commit point.
2108         */
2109        for (i = 0; i < group_size; i++) {
2110                tc = flex_array_get(group, i);
2111                cgroup_task_migrate(tc->cgrp, tc->task, tc->cset);
2112        }
2113        /* nothing is sensitive to fork() after this point. */
2114
2115        /*
2116         * step 4: do subsystem attach callbacks.
2117         */
2118        for_each_root_subsys(root, ss) {
2119                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
2120
2121                if (ss->attach)
2122                        ss->attach(css, &tset);
2123        }
2124
2125        /*
2126         * step 5: success! and cleanup
2127         */
2128        retval = 0;
2129out_put_css_set_refs:
2130        if (retval) {
2131                for (i = 0; i < group_size; i++) {
2132                        tc = flex_array_get(group, i);
2133                        if (!tc->cset)
2134                                break;
2135                        put_css_set(tc->cset);
2136                }
2137        }
2138out_cancel_attach:
2139        if (retval) {
2140                for_each_root_subsys(root, ss) {
2141                        struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
2142
2143                        if (ss == failed_ss)
2144                                break;
2145                        if (ss->cancel_attach)
2146                                ss->cancel_attach(css, &tset);
2147                }
2148        }
2149out_free_group_list:
2150        flex_array_free(group);
2151        return retval;
2152}
2153
2154/*
2155 * Find the task_struct of the task to attach by vpid and pass it along to the
2156 * function to attach either it or all tasks in its threadgroup. Will lock
2157 * cgroup_mutex and threadgroup; may take task_lock of task.
2158 */
2159static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup)
2160{
2161        struct task_struct *tsk;
2162        const struct cred *cred = current_cred(), *tcred;
2163        int ret;
2164
2165        if (!cgroup_lock_live_group(cgrp))
2166                return -ENODEV;
2167
2168retry_find_task:
2169        rcu_read_lock();
2170        if (pid) {
2171                tsk = find_task_by_vpid(pid);
2172                if (!tsk) {
2173                        rcu_read_unlock();
2174                        ret= -ESRCH;
2175                        goto out_unlock_cgroup;
2176                }
2177                /*
2178                 * even if we're attaching all tasks in the thread group, we
2179                 * only need to check permissions on one of them.
2180                 */
2181                tcred = __task_cred(tsk);
2182                if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
2183                    !uid_eq(cred->euid, tcred->uid) &&
2184                    !uid_eq(cred->euid, tcred->suid)) {
2185                        rcu_read_unlock();
2186                        ret = -EACCES;
2187                        goto out_unlock_cgroup;
2188                }
2189        } else
2190                tsk = current;
2191
2192        if (threadgroup)
2193                tsk = tsk->group_leader;
2194
2195        /*
2196         * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2197         * trapped in a cpuset, or RT worker may be born in a cgroup
2198         * with no rt_runtime allocated.  Just say no.
2199         */
2200        if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2201                ret = -EINVAL;
2202                rcu_read_unlock();
2203                goto out_unlock_cgroup;
2204        }
2205
2206        get_task_struct(tsk);
2207        rcu_read_unlock();
2208
2209        threadgroup_lock(tsk);
2210        if (threadgroup) {
2211                if (!thread_group_leader(tsk)) {
2212                        /*
2213                         * a race with de_thread from another thread's exec()
2214                         * may strip us of our leadership, if this happens,
2215                         * there is no choice but to throw this task away and
2216                         * try again; this is
2217                         * "double-double-toil-and-trouble-check locking".
2218                         */
2219                        threadgroup_unlock(tsk);
2220                        put_task_struct(tsk);
2221                        goto retry_find_task;
2222                }
2223        }
2224
2225        ret = cgroup_attach_task(cgrp, tsk, threadgroup);
2226
2227        threadgroup_unlock(tsk);
2228
2229        put_task_struct(tsk);
2230out_unlock_cgroup:
2231        mutex_unlock(&cgroup_mutex);
2232        return ret;
2233}
2234
2235/**
2236 * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
2237 * @from: attach to all cgroups of a given task
2238 * @tsk: the task to be attached
2239 */
2240int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
2241{
2242        struct cgroupfs_root *root;
2243        int retval = 0;
2244
2245        mutex_lock(&cgroup_mutex);
2246        for_each_active_root(root) {
2247                struct cgroup *from_cgrp = task_cgroup_from_root(from, root);
2248
2249                retval = cgroup_attach_task(from_cgrp, tsk, false);
2250                if (retval)
2251                        break;
2252        }
2253        mutex_unlock(&cgroup_mutex);
2254
2255        return retval;
2256}
2257EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
2258
2259static int cgroup_tasks_write(struct cgroup_subsys_state *css,
2260                              struct cftype *cft, u64 pid)
2261{
2262        return attach_task_by_pid(css->cgroup, pid, false);
2263}
2264
2265static int cgroup_procs_write(struct cgroup_subsys_state *css,
2266                              struct cftype *cft, u64 tgid)
2267{
2268        return attach_task_by_pid(css->cgroup, tgid, true);
2269}
2270
2271static int cgroup_release_agent_write(struct cgroup_subsys_state *css,
2272                                      struct cftype *cft, const char *buffer)
2273{
2274        BUILD_BUG_ON(sizeof(css->cgroup->root->release_agent_path) < PATH_MAX);
2275        if (strlen(buffer) >= PATH_MAX)
2276                return -EINVAL;
2277        if (!cgroup_lock_live_group(css->cgroup))
2278                return -ENODEV;
2279        mutex_lock(&cgroup_root_mutex);
2280        strcpy(css->cgroup->root->release_agent_path, buffer);
2281        mutex_unlock(&cgroup_root_mutex);
2282        mutex_unlock(&cgroup_mutex);
2283        return 0;
2284}
2285
2286static int cgroup_release_agent_show(struct cgroup_subsys_state *css,
2287                                     struct cftype *cft, struct seq_file *seq)
2288{
2289        struct cgroup *cgrp = css->cgroup;
2290
2291        if (!cgroup_lock_live_group(cgrp))
2292                return -ENODEV;
2293        seq_puts(seq, cgrp->root->release_agent_path);
2294        seq_putc(seq, '\n');
2295        mutex_unlock(&cgroup_mutex);
2296        return 0;
2297}
2298
2299static int cgroup_sane_behavior_show(struct cgroup_subsys_state *css,
2300                                     struct cftype *cft, struct seq_file *seq)
2301{
2302        seq_printf(seq, "%d\n", cgroup_sane_behavior(css->cgroup));
2303        return 0;
2304}
2305
2306/* A buffer size big enough for numbers or short strings */
2307#define CGROUP_LOCAL_BUFFER_SIZE 64
2308
2309static ssize_t cgroup_write_X64(struct cgroup_subsys_state *css,
2310                                struct cftype *cft, struct file *file,
2311                                const char __user *userbuf, size_t nbytes,
2312                                loff_t *unused_ppos)
2313{
2314        char buffer[CGROUP_LOCAL_BUFFER_SIZE];
2315        int retval = 0;
2316        char *end;
2317
2318        if (!nbytes)
2319                return -EINVAL;
2320        if (nbytes >= sizeof(buffer))
2321                return -E2BIG;
2322        if (copy_from_user(buffer, userbuf, nbytes))
2323                return -EFAULT;
2324
2325        buffer[nbytes] = 0;     /* nul-terminate */
2326        if (cft->write_u64) {
2327                u64 val = simple_strtoull(strstrip(buffer), &end, 0);
2328                if (*end)
2329                        return -EINVAL;
2330                retval = cft->write_u64(css, cft, val);
2331        } else {
2332                s64 val = simple_strtoll(strstrip(buffer), &end, 0);
2333                if (*end)
2334                        return -EINVAL;
2335                retval = cft->write_s64(css, cft, val);
2336        }
2337        if (!retval)
2338                retval = nbytes;
2339        return retval;
2340}
2341
2342static ssize_t cgroup_write_string(struct cgroup_subsys_state *css,
2343                                   struct cftype *cft, struct file *file,
2344                                   const char __user *userbuf, size_t nbytes,
2345                                   loff_t *unused_ppos)
2346{
2347        char local_buffer[CGROUP_LOCAL_BUFFER_SIZE];
2348        int retval = 0;
2349        size_t max_bytes = cft->max_write_len;
2350        char *buffer = local_buffer;
2351
2352        if (!max_bytes)
2353                max_bytes = sizeof(local_buffer) - 1;
2354        if (nbytes >= max_bytes)
2355                return -E2BIG;
2356        /* Allocate a dynamic buffer if we need one */
2357        if (nbytes >= sizeof(local_buffer)) {
2358                buffer = kmalloc(nbytes + 1, GFP_KERNEL);
2359                if (buffer == NULL)
2360                        return -ENOMEM;
2361        }
2362        if (nbytes && copy_from_user(buffer, userbuf, nbytes)) {
2363                retval = -EFAULT;
2364                goto out;
2365        }
2366
2367        buffer[nbytes] = 0;     /* nul-terminate */
2368        retval = cft->write_string(css, cft, strstrip(buffer));
2369        if (!retval)
2370                retval = nbytes;
2371out:
2372        if (buffer != local_buffer)
2373                kfree(buffer);
2374        return retval;
2375}
2376
2377static ssize_t cgroup_file_write(struct file *file, const char __user *buf,
2378                                 size_t nbytes, loff_t *ppos)
2379{
2380        struct cfent *cfe = __d_cfe(file->f_dentry);
2381        struct cftype *cft = __d_cft(file->f_dentry);
2382        struct cgroup_subsys_state *css = cfe->css;
2383
2384        if (cft->write)
2385                return cft->write(css, cft, file, buf, nbytes, ppos);
2386        if (cft->write_u64 || cft->write_s64)
2387                return cgroup_write_X64(css, cft, file, buf, nbytes, ppos);
2388        if (cft->write_string)
2389                return cgroup_write_string(css, cft, file, buf, nbytes, ppos);
2390        if (cft->trigger) {
2391                int ret = cft->trigger(css, (unsigned int)cft->private);
2392                return ret ? ret : nbytes;
2393        }
2394        return -EINVAL;
2395}
2396
2397static ssize_t cgroup_read_u64(struct cgroup_subsys_state *css,
2398                               struct cftype *cft, struct file *file,
2399                               char __user *buf, size_t nbytes, loff_t *ppos)
2400{
2401        char tmp[CGROUP_LOCAL_BUFFER_SIZE];
2402        u64 val = cft->read_u64(css, cft);
2403        int len = sprintf(tmp, "%llu\n", (unsigned long long) val);
2404
2405        return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
2406}
2407
2408static ssize_t cgroup_read_s64(struct cgroup_subsys_state *css,
2409                               struct cftype *cft, struct file *file,
2410                               char __user *buf, size_t nbytes, loff_t *ppos)
2411{
2412        char tmp[CGROUP_LOCAL_BUFFER_SIZE];
2413        s64 val = cft->read_s64(css, cft);
2414        int len = sprintf(tmp, "%lld\n", (long long) val);
2415
2416        return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
2417}
2418
2419static ssize_t cgroup_file_read(struct file *file, char __user *buf,
2420                                size_t nbytes, loff_t *ppos)
2421{
2422        struct cfent *cfe = __d_cfe(file->f_dentry);
2423        struct cftype *cft = __d_cft(file->f_dentry);
2424        struct cgroup_subsys_state *css = cfe->css;
2425
2426        if (cft->read)
2427                return cft->read(css, cft, file, buf, nbytes, ppos);
2428        if (cft->read_u64)
2429                return cgroup_read_u64(css, cft, file, buf, nbytes, ppos);
2430        if (cft->read_s64)
2431                return cgroup_read_s64(css, cft, file, buf, nbytes, ppos);
2432        return -EINVAL;
2433}
2434
2435/*
2436 * seqfile ops/methods for returning structured data. Currently just
2437 * supports string->u64 maps, but can be extended in future.
2438 */
2439
2440static int cgroup_map_add(struct cgroup_map_cb *cb, const char *key, u64 value)
2441{
2442        struct seq_file *sf = cb->state;
2443        return seq_printf(sf, "%s %llu\n", key, (unsigned long long)value);
2444}
2445
2446static int cgroup_seqfile_show(struct seq_file *m, void *arg)
2447{
2448        struct cfent *cfe = m->private;
2449        struct cftype *cft = cfe->type;
2450        struct cgroup_subsys_state *css = cfe->css;
2451
2452        if (cft->read_map) {
2453                struct cgroup_map_cb cb = {
2454                        .fill = cgroup_map_add,
2455                        .state = m,
2456                };
2457                return cft->read_map(css, cft, &cb);
2458        }
2459        return cft->read_seq_string(css, cft, m);
2460}
2461
2462static const struct file_operations cgroup_seqfile_operations = {
2463        .read = seq_read,
2464        .write = cgroup_file_write,
2465        .llseek = seq_lseek,
2466        .release = single_release,
2467};
2468
2469static int cgroup_file_open(struct inode *inode, struct file *file)
2470{
2471        struct cfent *cfe = __d_cfe(file->f_dentry);
2472        struct cftype *cft = __d_cft(file->f_dentry);
2473        struct cgroup *cgrp = __d_cgrp(cfe->dentry->d_parent);
2474        struct cgroup_subsys_state *css;
2475        int err;
2476
2477        err = generic_file_open(inode, file);
2478        if (err)
2479                return err;
2480
2481        /*
2482         * If the file belongs to a subsystem, pin the css.  Will be
2483         * unpinned either on open failure or release.  This ensures that
2484         * @css stays alive for all file operations.
2485         */
2486        rcu_read_lock();
2487        css = cgroup_css(cgrp, cft->ss);
2488        if (cft->ss && !css_tryget(css))
2489                css = NULL;
2490        rcu_read_unlock();
2491
2492        if (!css)
2493                return -ENODEV;
2494
2495        /*
2496         * @cfe->css is used by read/write/close to determine the
2497         * associated css.  @file->private_data would be a better place but
2498         * that's already used by seqfile.  Multiple accessors may use it
2499         * simultaneously which is okay as the association never changes.
2500         */
2501        WARN_ON_ONCE(cfe->css && cfe->css != css);
2502        cfe->css = css;
2503
2504        if (cft->read_map || cft->read_seq_string) {
2505                file->f_op = &cgroup_seqfile_operations;
2506                err = single_open(file, cgroup_seqfile_show, cfe);
2507        } else if (cft->open) {
2508                err = cft->open(inode, file);
2509        }
2510
2511        if (css->ss && err)
2512                css_put(css);
2513        return err;
2514}
2515
2516static int cgroup_file_release(struct inode *inode, struct file *file)
2517{
2518        struct cfent *cfe = __d_cfe(file->f_dentry);
2519        struct cftype *cft = __d_cft(file->f_dentry);
2520        struct cgroup_subsys_state *css = cfe->css;
2521        int ret = 0;
2522
2523        if (cft->release)
2524                ret = cft->release(inode, file);
2525        if (css->ss)
2526                css_put(css);
2527        return ret;
2528}
2529
2530/*
2531 * cgroup_rename - Only allow simple rename of directories in place.
2532 */
2533static int cgroup_rename(struct inode *old_dir, struct dentry *old_dentry,
2534                            struct inode *new_dir, struct dentry *new_dentry)
2535{
2536        int ret;
2537        struct cgroup_name *name, *old_name;
2538        struct cgroup *cgrp;
2539
2540        /*
2541         * It's convinient to use parent dir's i_mutex to protected
2542         * cgrp->name.
2543         */
2544        lockdep_assert_held(&old_dir->i_mutex);
2545
2546        if (!S_ISDIR(old_dentry->d_inode->i_mode))
2547                return -ENOTDIR;
2548        if (new_dentry->d_inode)
2549                return -EEXIST;
2550        if (old_dir != new_dir)
2551                return -EIO;
2552
2553        cgrp = __d_cgrp(old_dentry);
2554
2555        /*
2556         * This isn't a proper migration and its usefulness is very
2557         * limited.  Disallow if sane_behavior.
2558         */
2559        if (cgroup_sane_behavior(cgrp))
2560                return -EPERM;
2561
2562        name = cgroup_alloc_name(new_dentry);
2563        if (!name)
2564                return -ENOMEM;
2565
2566        ret = simple_rename(old_dir, old_dentry, new_dir, new_dentry);
2567        if (ret) {
2568                kfree(name);
2569                return ret;
2570        }
2571
2572        old_name = rcu_dereference_protected(cgrp->name, true);
2573        rcu_assign_pointer(cgrp->name, name);
2574
2575        kfree_rcu(old_name, rcu_head);
2576        return 0;
2577}
2578
2579static struct simple_xattrs *__d_xattrs(struct dentry *dentry)
2580{
2581        if (S_ISDIR(dentry->d_inode->i_mode))
2582                return &__d_cgrp(dentry)->xattrs;
2583        else
2584                return &__d_cfe(dentry)->xattrs;
2585}
2586
2587static inline int xattr_enabled(struct dentry *dentry)
2588{
2589        struct cgroupfs_root *root = dentry->d_sb->s_fs_info;
2590        return root->flags & CGRP_ROOT_XATTR;
2591}
2592
2593static bool is_valid_xattr(const char *name)
2594{
2595        if (!strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) ||
2596            !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN))
2597                return true;
2598        return false;
2599}
2600
2601static int cgroup_setxattr(struct dentry *dentry, const char *name,
2602                           const void *val, size_t size, int flags)
2603{
2604        if (!xattr_enabled(dentry))
2605                return -EOPNOTSUPP;
2606        if (!is_valid_xattr(name))
2607                return -EINVAL;
2608        return simple_xattr_set(__d_xattrs(dentry), name, val, size, flags);
2609}
2610
2611static int cgroup_removexattr(struct dentry *dentry, const char *name)
2612{
2613        if (!xattr_enabled(dentry))
2614                return -EOPNOTSUPP;
2615        if (!is_valid_xattr(name))
2616                return -EINVAL;
2617        return simple_xattr_remove(__d_xattrs(dentry), name);
2618}
2619
2620static ssize_t cgroup_getxattr(struct dentry *dentry, const char *name,
2621                               void *buf, size_t size)
2622{
2623        if (!xattr_enabled(dentry))
2624                return -EOPNOTSUPP;
2625        if (!is_valid_xattr(name))
2626                return -EINVAL;
2627        return simple_xattr_get(__d_xattrs(dentry), name, buf, size);
2628}
2629
2630static ssize_t cgroup_listxattr(struct dentry *dentry, char *buf, size_t size)
2631{
2632        if (!xattr_enabled(dentry))
2633                return -EOPNOTSUPP;
2634        return simple_xattr_list(__d_xattrs(dentry), buf, size);
2635}
2636
2637static const struct file_operations cgroup_file_operations = {
2638        .read = cgroup_file_read,
2639        .write = cgroup_file_write,
2640        .llseek = generic_file_llseek,
2641        .open = cgroup_file_open,
2642        .release = cgroup_file_release,
2643};
2644
2645static const struct inode_operations cgroup_file_inode_operations = {
2646        .setxattr = cgroup_setxattr,
2647        .getxattr = cgroup_getxattr,
2648        .listxattr = cgroup_listxattr,
2649        .removexattr = cgroup_removexattr,
2650};
2651
2652static const struct inode_operations cgroup_dir_inode_operations = {
2653        .lookup = simple_lookup,
2654        .mkdir = cgroup_mkdir,
2655        .rmdir = cgroup_rmdir,
2656        .rename = cgroup_rename,
2657        .setxattr = cgroup_setxattr,
2658        .getxattr = cgroup_getxattr,
2659        .listxattr = cgroup_listxattr,
2660        .removexattr = cgroup_removexattr,
2661};
2662
2663/*
2664 * Check if a file is a control file
2665 */
2666static inline struct cftype *__file_cft(struct file *file)
2667{
2668        if (file_inode(file)->i_fop != &cgroup_file_operations)
2669                return ERR_PTR(-EINVAL);
2670        return __d_cft(file->f_dentry);
2671}
2672
2673static int cgroup_create_file(struct dentry *dentry, umode_t mode,
2674                                struct super_block *sb)
2675{
2676        struct inode *inode;
2677
2678        if (!dentry)
2679                return -ENOENT;
2680        if (dentry->d_inode)
2681                return -EEXIST;
2682
2683        inode = cgroup_new_inode(mode, sb);
2684        if (!inode)
2685                return -ENOMEM;
2686
2687        if (S_ISDIR(mode)) {
2688                inode->i_op = &cgroup_dir_inode_operations;
2689                inode->i_fop = &simple_dir_operations;
2690
2691                /* start off with i_nlink == 2 (for "." entry) */
2692                inc_nlink(inode);
2693                inc_nlink(dentry->d_parent->d_inode);
2694
2695                /*
2696                 * Control reaches here with cgroup_mutex held.
2697                 * @inode->i_mutex should nest outside cgroup_mutex but we
2698                 * want to populate it immediately without releasing
2699                 * cgroup_mutex.  As @inode isn't visible to anyone else
2700                 * yet, trylock will always succeed without affecting
2701                 * lockdep checks.
2702                 */
2703                WARN_ON_ONCE(!mutex_trylock(&inode->i_mutex));
2704        } else if (S_ISREG(mode)) {
2705                inode->i_size = 0;
2706                inode->i_fop = &cgroup_file_operations;
2707                inode->i_op = &cgroup_file_inode_operations;
2708        }
2709        d_instantiate(dentry, inode);
2710        dget(dentry);   /* Extra count - pin the dentry in core */
2711        return 0;
2712}
2713
2714/**
2715 * cgroup_file_mode - deduce file mode of a control file
2716 * @cft: the control file in question
2717 *
2718 * returns cft->mode if ->mode is not 0
2719 * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
2720 * returns S_IRUGO if it has only a read handler
2721 * returns S_IWUSR if it has only a write hander
2722 */
2723static umode_t cgroup_file_mode(const struct cftype *cft)
2724{
2725        umode_t mode = 0;
2726
2727        if (cft->mode)
2728                return cft->mode;
2729
2730        if (cft->read || cft->read_u64 || cft->read_s64 ||
2731            cft->read_map || cft->read_seq_string)
2732                mode |= S_IRUGO;
2733
2734        if (cft->write || cft->write_u64 || cft->write_s64 ||
2735            cft->write_string || cft->trigger)
2736                mode |= S_IWUSR;
2737
2738        return mode;
2739}
2740
2741static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
2742{
2743        struct dentry *dir = cgrp->dentry;
2744        struct cgroup *parent = __d_cgrp(dir);
2745        struct dentry *dentry;
2746        struct cfent *cfe;
2747        int error;
2748        umode_t mode;
2749        char name[MAX_CGROUP_TYPE_NAMELEN + MAX_CFTYPE_NAME + 2] = { 0 };
2750
2751        if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
2752            !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {
2753                strcpy(name, cft->ss->name);
2754                strcat(name, ".");
2755        }
2756        strcat(name, cft->name);
2757
2758        BUG_ON(!mutex_is_locked(&dir->d_inode->i_mutex));
2759
2760        cfe = kzalloc(sizeof(*cfe), GFP_KERNEL);
2761        if (!cfe)
2762                return -ENOMEM;
2763
2764        dentry = lookup_one_len(name, dir, strlen(name));
2765        if (IS_ERR(dentry)) {
2766                error = PTR_ERR(dentry);
2767                goto out;
2768        }
2769
2770        cfe->type = (void *)cft;
2771        cfe->dentry = dentry;
2772        dentry->d_fsdata = cfe;
2773        simple_xattrs_init(&cfe->xattrs);
2774
2775        mode = cgroup_file_mode(cft);
2776        error = cgroup_create_file(dentry, mode | S_IFREG, cgrp->root->sb);
2777        if (!error) {
2778                list_add_tail(&cfe->node, &parent->files);
2779                cfe = NULL;
2780        }
2781        dput(dentry);
2782out:
2783        kfree(cfe);
2784        return error;
2785}
2786
2787/**
2788 * cgroup_addrm_files - add or remove files to a cgroup directory
2789 * @cgrp: the target cgroup
2790 * @cfts: array of cftypes to be added
2791 * @is_add: whether to add or remove
2792 *
2793 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
2794 * For removals, this function never fails.  If addition fails, this
2795 * function doesn't remove files already added.  The caller is responsible
2796 * for cleaning up.
2797 */
2798static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
2799                              bool is_add)
2800{
2801        struct cftype *cft;
2802        int ret;
2803
2804        lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
2805        lockdep_assert_held(&cgroup_mutex);
2806
2807        for (cft = cfts; cft->name[0] != '\0'; cft++) {
2808                /* does cft->flags tell us to skip this file on @cgrp? */
2809                if ((cft->flags & CFTYPE_INSANE) && cgroup_sane_behavior(cgrp))
2810                        continue;
2811                if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgrp->parent)
2812                        continue;
2813                if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgrp->parent)
2814                        continue;
2815
2816                if (is_add) {
2817                        ret = cgroup_add_file(cgrp, cft);
2818                        if (ret) {
2819                                pr_warn("cgroup_addrm_files: failed to add %s, err=%d\n",
2820                                        cft->name, ret);
2821                                return ret;
2822                        }
2823                } else {
2824                        cgroup_rm_file(cgrp, cft);
2825                }
2826        }
2827        return 0;
2828}
2829
2830static void cgroup_cfts_prepare(void)
2831        __acquires(&cgroup_mutex)
2832{
2833        /*
2834         * Thanks to the entanglement with vfs inode locking, we can't walk
2835         * the existing cgroups under cgroup_mutex and create files.
2836         * Instead, we use css_for_each_descendant_pre() and drop RCU read
2837         * lock before calling cgroup_addrm_files().
2838         */
2839        mutex_lock(&cgroup_mutex);
2840}
2841
2842static int cgroup_cfts_commit(struct cftype *cfts, bool is_add)
2843        __releases(&cgroup_mutex)
2844{
2845        LIST_HEAD(pending);
2846        struct cgroup_subsys *ss = cfts[0].ss;
2847        struct cgroup *root = &ss->root->top_cgroup;
2848        struct super_block *sb = ss->root->sb;
2849        struct dentry *prev = NULL;
2850        struct inode *inode;
2851        struct cgroup_subsys_state *css;
2852        u64 update_before;
2853        int ret = 0;
2854
2855        /* %NULL @cfts indicates abort and don't bother if @ss isn't attached */
2856        if (!cfts || ss->root == &cgroup_dummy_root ||
2857            !atomic_inc_not_zero(&sb->s_active)) {
2858                mutex_unlock(&cgroup_mutex);
2859                return 0;
2860        }
2861
2862        /*
2863         * All cgroups which are created after we drop cgroup_mutex will
2864         * have the updated set of files, so we only need to update the
2865         * cgroups created before the current @cgroup_serial_nr_next.
2866         */
2867        update_before = cgroup_serial_nr_next;
2868
2869        mutex_unlock(&cgroup_mutex);
2870
2871        /* add/rm files for all cgroups created before */
2872        rcu_read_lock();
2873        css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
2874                struct cgroup *cgrp = css->cgroup;
2875
2876                if (cgroup_is_dead(cgrp))
2877                        continue;
2878
2879                inode = cgrp->dentry->d_inode;
2880                dget(cgrp->dentry);
2881                rcu_read_unlock();
2882
2883                dput(prev);
2884                prev = cgrp->dentry;
2885
2886                mutex_lock(&inode->i_mutex);
2887                mutex_lock(&cgroup_mutex);
2888                if (cgrp->serial_nr < update_before && !cgroup_is_dead(cgrp))
2889                        ret = cgroup_addrm_files(cgrp, cfts, is_add);
2890                mutex_unlock(&cgroup_mutex);
2891                mutex_unlock(&inode->i_mutex);
2892
2893                rcu_read_lock();
2894                if (ret)
2895                        break;
2896        }
2897        rcu_read_unlock();
2898        dput(prev);
2899        deactivate_super(sb);
2900        return ret;
2901}
2902
2903/**
2904 * cgroup_add_cftypes - add an array of cftypes to a subsystem
2905 * @ss: target cgroup subsystem
2906 * @cfts: zero-length name terminated array of cftypes
2907 *
2908 * Register @cfts to @ss.  Files described by @cfts are created for all
2909 * existing cgroups to which @ss is attached and all future cgroups will
2910 * have them too.  This function can be called anytime whether @ss is
2911 * attached or not.
2912 *
2913 * Returns 0 on successful registration, -errno on failure.  Note that this
2914 * function currently returns 0 as long as @cfts registration is successful
2915 * even if some file creation attempts on existing cgroups fail.
2916 */
2917int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
2918{
2919        struct cftype_set *set;
2920        struct cftype *cft;
2921        int ret;
2922
2923        set = kzalloc(sizeof(*set), GFP_KERNEL);
2924        if (!set)
2925                return -ENOMEM;
2926
2927        for (cft = cfts; cft->name[0] != '\0'; cft++)
2928                cft->ss = ss;
2929
2930        cgroup_cfts_prepare();
2931        set->cfts = cfts;
2932        list_add_tail(&set->node, &ss->cftsets);
2933        ret = cgroup_cfts_commit(cfts, true);
2934        if (ret)
2935                cgroup_rm_cftypes(cfts);
2936        return ret;
2937}
2938EXPORT_SYMBOL_GPL(cgroup_add_cftypes);
2939
2940/**
2941 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
2942 * @cfts: zero-length name terminated array of cftypes
2943 *
2944 * Unregister @cfts.  Files described by @cfts are removed from all
2945 * existing cgroups and all future cgroups won't have them either.  This
2946 * function can be called anytime whether @cfts' subsys is attached or not.
2947 *
2948 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
2949 * registered.
2950 */
2951int cgroup_rm_cftypes(struct cftype *cfts)
2952{
2953        struct cftype_set *set;
2954
2955        if (!cfts || !cfts[0].ss)
2956                return -ENOENT;
2957
2958        cgroup_cfts_prepare();
2959
2960        list_for_each_entry(set, &cfts[0].ss->cftsets, node) {
2961                if (set->cfts == cfts) {
2962                        list_del(&set->node);
2963                        kfree(set);
2964                        cgroup_cfts_commit(cfts, false);
2965                        return 0;
2966                }
2967        }
2968
2969        cgroup_cfts_commit(NULL, false);
2970        return -ENOENT;
2971}
2972
2973/**
2974 * cgroup_task_count - count the number of tasks in a cgroup.
2975 * @cgrp: the cgroup in question
2976 *
2977 * Return the number of tasks in the cgroup.
2978 */
2979int cgroup_task_count(const struct cgroup *cgrp)
2980{
2981        int count = 0;
2982        struct cgrp_cset_link *link;
2983
2984        read_lock(&css_set_lock);
2985        list_for_each_entry(link, &cgrp->cset_links, cset_link)
2986                count += atomic_read(&link->cset->refcount);
2987        read_unlock(&css_set_lock);
2988        return count;
2989}
2990
2991/*
2992 * To reduce the fork() overhead for systems that are not actually using
2993 * their cgroups capability, we don't maintain the lists running through
2994 * each css_set to its tasks until we see the list actually used - in other
2995 * words after the first call to css_task_iter_start().
2996 */
2997static void cgroup_enable_task_cg_lists(void)
2998{
2999        struct task_struct *p, *g;
3000        write_lock(&css_set_lock);
3001        use_task_css_set_links = 1;
3002        /*
3003         * We need tasklist_lock because RCU is not safe against
3004         * while_each_thread(). Besides, a forking task that has passed
3005         * cgroup_post_fork() without seeing use_task_css_set_links = 1
3006         * is not guaranteed to have its child immediately visible in the
3007         * tasklist if we walk through it with RCU.
3008         */
3009        read_lock(&tasklist_lock);
3010        do_each_thread(g, p) {
3011                task_lock(p);
3012                /*
3013                 * We should check if the process is exiting, otherwise
3014                 * it will race with cgroup_exit() in that the list
3015                 * entry won't be deleted though the process has exited.
3016                 */
3017                if (!(p->flags & PF_EXITING) && list_empty(&p->cg_list))
3018                        list_add(&p->cg_list, &task_css_set(p)->tasks);
3019                task_unlock(p);
3020        } while_each_thread(g, p);
3021        read_unlock(&tasklist_lock);
3022        write_unlock(&css_set_lock);
3023}
3024
3025/**
3026 * css_next_child - find the next child of a given css
3027 * @pos_css: the current position (%NULL to initiate traversal)
3028 * @parent_css: css whose children to walk
3029 *
3030 * This function returns the next child of @parent_css and should be called
3031 * under RCU read lock.  The only requirement is that @parent_css and
3032 * @pos_css are accessible.  The next sibling is guaranteed to be returned
3033 * regardless of their states.
3034 */
3035struct cgroup_subsys_state *
3036css_next_child(struct cgroup_subsys_state *pos_css,
3037               struct cgroup_subsys_state *parent_css)
3038{
3039        struct cgroup *pos = pos_css ? pos_css->cgroup : NULL;
3040        struct cgroup *cgrp = parent_css->cgroup;
3041        struct cgroup *next;
3042
3043        WARN_ON_ONCE(!rcu_read_lock_held());
3044
3045        /*
3046         * @pos could already have been removed.  Once a cgroup is removed,
3047         * its ->sibling.next is no longer updated when its next sibling
3048         * changes.  As CGRP_DEAD assertion is serialized and happens
3049         * before the cgroup is taken off the ->sibling list, if we see it
3050         * unasserted, it's guaranteed that the next sibling hasn't
3051         * finished its grace period even if it's already removed, and thus
3052         * safe to dereference from this RCU critical section.  If
3053         * ->sibling.next is inaccessible, cgroup_is_dead() is guaranteed
3054         * to be visible as %true here.
3055         *
3056         * If @pos is dead, its next pointer can't be dereferenced;
3057         * however, as each cgroup is given a monotonically increasing
3058         * unique serial number and always appended to the sibling list,
3059         * the next one can be found by walking the parent's children until
3060         * we see a cgroup with higher serial number than @pos's.  While
3061         * this path can be slower, it's taken only when either the current
3062         * cgroup is removed or iteration and removal race.
3063         */
3064        if (!pos) {
3065                next = list_entry_rcu(cgrp->children.next, struct cgroup, sibling);
3066        } else if (likely(!cgroup_is_dead(pos))) {
3067                next = list_entry_rcu(pos->sibling.next, struct cgroup, sibling);
3068        } else {
3069                list_for_each_entry_rcu(next, &cgrp->children, sibling)
3070                        if (next->serial_nr > pos->serial_nr)
3071                                break;
3072        }
3073
3074        if (&next->sibling == &cgrp->children)
3075                return NULL;
3076
3077        return cgroup_css(next, parent_css->ss);
3078}
3079EXPORT_SYMBOL_GPL(css_next_child);
3080
3081/**
3082 * css_next_descendant_pre - find the next descendant for pre-order walk
3083 * @pos: the current position (%NULL to initiate traversal)
3084 * @root: css whose descendants to walk
3085 *
3086 * To be used by css_for_each_descendant_pre().  Find the next descendant
3087 * to visit for pre-order traversal of @root's descendants.  @root is
3088 * included in the iteration and the first node to be visited.
3089 *
3090 * While this function requires RCU read locking, it doesn't require the
3091 * whole traversal to be contained in a single RCU critical section.  This
3092 * function will return the correct next descendant as long as both @pos
3093 * and @root are accessible and @pos is a descendant of @root.
3094 */
3095struct cgroup_subsys_state *
3096css_next_descendant_pre(struct cgroup_subsys_state *pos,
3097                        struct cgroup_subsys_state *root)
3098{
3099        struct cgroup_subsys_state *next;
3100
3101        WARN_ON_ONCE(!rcu_read_lock_held());
3102
3103        /* if first iteration, visit @root */
3104        if (!pos)
3105                return root;
3106
3107        /* visit the first child if exists */
3108        next = css_next_child(NULL, pos);
3109        if (next)
3110                return next;
3111
3112        /* no child, visit my or the closest ancestor's next sibling */
3113        while (pos != root) {
3114                next = css_next_child(pos, css_parent(pos));
3115                if (next)
3116                        return next;
3117                pos = css_parent(pos);
3118        }
3119
3120        return NULL;
3121}
3122EXPORT_SYMBOL_GPL(css_next_descendant_pre);
3123
3124/**
3125 * css_rightmost_descendant - return the rightmost descendant of a css
3126 * @pos: css of interest
3127 *
3128 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
3129 * is returned.  This can be used during pre-order traversal to skip
3130 * subtree of @pos.
3131 *
3132 * While this function requires RCU read locking, it doesn't require the
3133 * whole traversal to be contained in a single RCU critical section.  This
3134 * function will return the correct rightmost descendant as long as @pos is
3135 * accessible.
3136 */
3137struct cgroup_subsys_state *
3138css_rightmost_descendant(struct cgroup_subsys_state *pos)
3139{
3140        struct cgroup_subsys_state *last, *tmp;
3141
3142        WARN_ON_ONCE(!rcu_read_lock_held());
3143
3144        do {
3145                last = pos;
3146                /* ->prev isn't RCU safe, walk ->next till the end */
3147                pos = NULL;
3148                css_for_each_child(tmp, last)
3149                        pos = tmp;
3150        } while (pos);
3151
3152        return last;
3153}
3154EXPORT_SYMBOL_GPL(css_rightmost_descendant);
3155
3156static struct cgroup_subsys_state *
3157css_leftmost_descendant(struct cgroup_subsys_state *pos)
3158{
3159        struct cgroup_subsys_state *last;
3160
3161        do {
3162                last = pos;
3163                pos = css_next_child(NULL, pos);
3164        } while (pos);
3165
3166        return last;
3167}
3168
3169/**
3170 * css_next_descendant_post - find the next descendant for post-order walk
3171 * @pos: the current position (%NULL to initiate traversal)
3172 * @root: css whose descendants to walk
3173 *
3174 * To be used by css_for_each_descendant_post().  Find the next descendant
3175 * to visit for post-order traversal of @root's descendants.  @root is
3176 * included in the iteration and the last node to be visited.
3177 *
3178 * While this function requires RCU read locking, it doesn't require the
3179 * whole traversal to be contained in a single RCU critical section.  This
3180 * function will return the correct next descendant as long as both @pos
3181 * and @cgroup are accessible and @pos is a descendant of @cgroup.
3182 */
3183struct cgroup_subsys_state *
3184css_next_descendant_post(struct cgroup_subsys_state *pos,
3185                         struct cgroup_subsys_state *root)
3186{
3187        struct cgroup_subsys_state *next;
3188
3189        WARN_ON_ONCE(!rcu_read_lock_held());
3190
3191        /* if first iteration, visit leftmost descendant which may be @root */
3192        if (!pos)
3193                return css_leftmost_descendant(root);
3194
3195        /* if we visited @root, we're done */
3196        if (pos == root)
3197                return NULL;
3198
3199        /* if there's an unvisited sibling, visit its leftmost descendant */
3200        next = css_next_child(pos, css_parent(pos));
3201        if (next)
3202                return css_leftmost_descendant(next);
3203
3204        /* no sibling left, visit parent */
3205        return css_parent(pos);
3206}
3207EXPORT_SYMBOL_GPL(css_next_descendant_post);
3208
3209/**
3210 * css_advance_task_iter - advance a task itererator to the next css_set
3211 * @it: the iterator to advance
3212 *
3213 * Advance @it to the next css_set to walk.
3214 */
3215static void css_advance_task_iter(struct css_task_iter *it)
3216{
3217        struct list_head *l = it->cset_link;
3218        struct cgrp_cset_link *link;
3219        struct css_set *cset;
3220
3221        /* Advance to the next non-empty css_set */
3222        do {
3223                l = l->next;
3224                if (l == &it->origin_css->cgroup->cset_links) {
3225                        it->cset_link = NULL;
3226                        return;
3227                }
3228                link = list_entry(l, struct cgrp_cset_link, cset_link);
3229                cset = link->cset;
3230        } while (list_empty(&cset->tasks));
3231        it->cset_link = l;
3232        it->task = cset->tasks.next;
3233}
3234
3235/**
3236 * css_task_iter_start - initiate task iteration
3237 * @css: the css to walk tasks of
3238 * @it: the task iterator to use
3239 *
3240 * Initiate iteration through the tasks of @css.  The caller can call
3241 * css_task_iter_next() to walk through the tasks until the function
3242 * returns NULL.  On completion of iteration, css_task_iter_end() must be
3243 * called.
3244 *
3245 * Note that this function acquires a lock which is released when the
3246 * iteration finishes.  The caller can't sleep while iteration is in
3247 * progress.
3248 */
3249void css_task_iter_start(struct cgroup_subsys_state *css,
3250                         struct css_task_iter *it)
3251        __acquires(css_set_lock)
3252{
3253        /*
3254         * The first time anyone tries to iterate across a css, we need to
3255         * enable the list linking each css_set to its tasks, and fix up
3256         * all existing tasks.
3257         */
3258        if (!use_task_css_set_links)
3259                cgroup_enable_task_cg_lists();
3260
3261        read_lock(&css_set_lock);
3262
3263        it->origin_css = css;
3264        it->cset_link = &css->cgroup->cset_links;
3265
3266        css_advance_task_iter(it);
3267}
3268
3269/**
3270 * css_task_iter_next - return the next task for the iterator
3271 * @it: the task iterator being iterated
3272 *
3273 * The "next" function for task iteration.  @it should have been
3274 * initialized via css_task_iter_start().  Returns NULL when the iteration
3275 * reaches the end.
3276 */
3277struct task_struct *css_task_iter_next(struct css_task_iter *it)
3278{
3279        struct task_struct *res;
3280        struct list_head *l = it->task;
3281        struct cgrp_cset_link *link;
3282
3283        /* If the iterator cg is NULL, we have no tasks */
3284        if (!it->cset_link)
3285                return NULL;
3286        res = list_entry(l, struct task_struct, cg_list);
3287        /* Advance iterator to find next entry */
3288        l = l->next;
3289        link = list_entry(it->cset_link, struct cgrp_cset_link, cset_link);
3290        if (l == &link->cset->tasks) {
3291                /*
3292                 * We reached the end of this task list - move on to the
3293                 * next cgrp_cset_link.
3294                 */
3295                css_advance_task_iter(it);
3296        } else {
3297                it->task = l;
3298        }
3299        return res;
3300}
3301
3302/**
3303 * css_task_iter_end - finish task iteration
3304 * @it: the task iterator to finish
3305 *
3306 * Finish task iteration started by css_task_iter_start().
3307 */
3308void css_task_iter_end(struct css_task_iter *it)
3309        __releases(css_set_lock)
3310{
3311        read_unlock(&css_set_lock);
3312}
3313
3314static inline int started_after_time(struct task_struct *t1,
3315                                     struct timespec *time,
3316                                     struct task_struct *t2)
3317{
3318        int start_diff = timespec_compare(&t1->start_time, time);
3319        if (start_diff > 0) {
3320                return 1;
3321        } else if (start_diff < 0) {
3322                return 0;
3323        } else {
3324                /*
3325                 * Arbitrarily, if two processes started at the same
3326                 * time, we'll say that the lower pointer value
3327                 * started first. Note that t2 may have exited by now
3328                 * so this may not be a valid pointer any longer, but
3329                 * that's fine - it still serves to distinguish
3330                 * between two tasks started (effectively) simultaneously.
3331                 */
3332                return t1 > t2;
3333        }
3334}
3335
3336/*
3337 * This function is a callback from heap_insert() and is used to order
3338 * the heap.
3339 * In this case we order the heap in descending task start time.
3340 */
3341static inline int started_after(void *p1, void *p2)
3342{
3343        struct task_struct *t1 = p1;
3344        struct task_struct *t2 = p2;
3345        return started_after_time(t1, &t2->start_time, t2);
3346}
3347
3348/**
3349 * css_scan_tasks - iterate though all the tasks in a css
3350 * @css: the css to iterate tasks of
3351 * @test: optional test callback
3352 * @process: process callback
3353 * @data: data passed to @test and @process
3354 * @heap: optional pre-allocated heap used for task iteration
3355 *
3356 * Iterate through all the tasks in @css, calling @test for each, and if it
3357 * returns %true, call @process for it also.
3358 *
3359 * @test may be NULL, meaning always true (select all tasks), which
3360 * effectively duplicates css_task_iter_{start,next,end}() but does not
3361 * lock css_set_lock for the call to @process.
3362 *
3363 * It is guaranteed that @process will act on every task that is a member
3364 * of @css for the duration of this call.  This function may or may not
3365 * call @process for tasks that exit or move to a different css during the
3366 * call, or are forked or move into the css during the call.
3367 *
3368 * Note that @test may be called with locks held, and may in some
3369 * situations be called multiple times for the same task, so it should be
3370 * cheap.
3371 *
3372 * If @heap is non-NULL, a heap has been pre-allocated and will be used for
3373 * heap operations (and its "gt" member will be overwritten), else a
3374 * temporary heap will be used (allocation of which may cause this function
3375 * to fail).
3376 */
3377int css_scan_tasks(struct cgroup_subsys_state *css,
3378                   bool (*test)(struct task_struct *, void *),
3379                   void (*process)(struct task_struct *, void *),
3380                   void *data, struct ptr_heap *heap)
3381{
3382        int retval, i;
3383        struct css_task_iter it;
3384        struct task_struct *p, *dropped;
3385        /* Never dereference latest_task, since it's not refcounted */
3386        struct task_struct *latest_task = NULL;
3387        struct ptr_heap tmp_heap;
3388        struct timespec latest_time = { 0, 0 };
3389
3390        if (heap) {
3391                /* The caller supplied our heap and pre-allocated its memory */
3392                heap->gt = &started_after;
3393        } else {
3394                /* We need to allocate our own heap memory */
3395                heap = &tmp_heap;
3396                retval = heap_init(heap, PAGE_SIZE, GFP_KERNEL, &started_after);
3397                if (retval)
3398                        /* cannot allocate the heap */
3399                        return retval;
3400        }
3401
3402 again:
3403        /*
3404         * Scan tasks in the css, using the @test callback to determine
3405         * which are of interest, and invoking @process callback on the
3406         * ones which need an update.  Since we don't want to hold any
3407         * locks during the task updates, gather tasks to be processed in a
3408         * heap structure.  The heap is sorted by descending task start
3409         * time.  If the statically-sized heap fills up, we overflow tasks
3410         * that started later, and in future iterations only consider tasks
3411         * that started after the latest task in the previous pass. This
3412         * guarantees forward progress and that we don't miss any tasks.
3413         */
3414        heap->size = 0;
3415        css_task_iter_start(css, &it);
3416        while ((p = css_task_iter_next(&it))) {
3417                /*
3418                 * Only affect tasks that qualify per the caller's callback,
3419                 * if he provided one
3420                 */
3421                if (test && !test(p, data))
3422                        continue;
3423                /*
3424                 * Only process tasks that started after the last task
3425                 * we processed
3426                 */
3427                if (!started_after_time(p, &latest_time, latest_task))
3428                        continue;
3429                dropped = heap_insert(heap, p);
3430                if (dropped == NULL) {
3431                        /*
3432                         * The new task was inserted; the heap wasn't
3433                         * previously full
3434                         */
3435                        get_task_struct(p);
3436                } else if (dropped != p) {
3437                        /*
3438                         * The new task was inserted, and pushed out a
3439                         * different task
3440                         */
3441                        get_task_struct(p);
3442                        put_task_struct(dropped);
3443                }
3444                /*
3445                 * Else the new task was newer than anything already in
3446                 * the heap and wasn't inserted
3447                 */
3448        }
3449        css_task_iter_end(&it);
3450
3451        if (heap->size) {
3452                for (i = 0; i < heap->size; i++) {
3453                        struct task_struct *q = heap->ptrs[i];
3454                        if (i == 0) {
3455                                latest_time = q->start_time;
3456                                latest_task = q;
3457                        }
3458                        /* Process the task per the caller's callback */
3459                        process(q, data);
3460                        put_task_struct(q);
3461                }
3462                /*
3463                 * If we had to process any tasks at all, scan again
3464                 * in case some of them were in the middle of forking
3465                 * children that didn't get processed.
3466                 * Not the most efficient way to do it, but it avoids
3467                 * having to take callback_mutex in the fork path
3468                 */
3469                goto again;
3470        }
3471        if (heap == &tmp_heap)
3472                heap_free(&tmp_heap);
3473        return 0;
3474}
3475
3476static void cgroup_transfer_one_task(struct task_struct *task, void *data)
3477{
3478        struct cgroup *new_cgroup = data;
3479
3480        mutex_lock(&cgroup_mutex);
3481        cgroup_attach_task(new_cgroup, task, false);
3482        mutex_unlock(&cgroup_mutex);
3483}
3484
3485/**
3486 * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
3487 * @to: cgroup to which the tasks will be moved
3488 * @from: cgroup in which the tasks currently reside
3489 */
3490int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
3491{
3492        return css_scan_tasks(&from->dummy_css, NULL, cgroup_transfer_one_task,
3493                              to, NULL);
3494}
3495
3496/*
3497 * Stuff for reading the 'tasks'/'procs' files.
3498 *
3499 * Reading this file can return large amounts of data if a cgroup has
3500 * *lots* of attached tasks. So it may need several calls to read(),
3501 * but we cannot guarantee that the information we produce is correct
3502 * unless we produce it entirely atomically.
3503 *
3504 */
3505
3506/* which pidlist file are we talking about? */
3507enum cgroup_filetype {
3508        CGROUP_FILE_PROCS,
3509        CGROUP_FILE_TASKS,
3510};
3511
3512/*
3513 * A pidlist is a list of pids that virtually represents the contents of one
3514 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
3515 * a pair (one each for procs, tasks) for each pid namespace that's relevant
3516 * to the cgroup.
3517 */
3518struct cgroup_pidlist {
3519        /*
3520         * used to find which pidlist is wanted. doesn't change as long as
3521         * this particular list stays in the list.
3522        */
3523        struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
3524        /* array of xids */
3525        pid_t *list;
3526        /* how many elements the above list has */
3527        int length;
3528        /* how many files are using the current array */
3529        int use_count;
3530        /* each of these stored in a list by its cgroup */
3531        struct list_head links;
3532        /* pointer to the cgroup we belong to, for list removal purposes */
3533        struct cgroup *owner;
3534        /* protects the other fields */
3535        struct rw_semaphore rwsem;
3536};
3537
3538/*
3539 * The following two functions "fix" the issue where there are more pids
3540 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
3541 * TODO: replace with a kernel-wide solution to this problem
3542 */
3543#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
3544static void *pidlist_allocate(int count)
3545{
3546        if (PIDLIST_TOO_LARGE(count))
3547                return vmalloc(count * sizeof(pid_t));
3548        else
3549                return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
3550}
3551static void pidlist_free(void *p)
3552{
3553        if (is_vmalloc_addr(p))
3554                vfree(p);
3555        else
3556                kfree(p);
3557}
3558
3559/*
3560 * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3561 * Returns the number of unique elements.
3562 */
3563static int pidlist_uniq(pid_t *list, int length)
3564{
3565        int src, dest = 1;
3566
3567        /*
3568         * we presume the 0th element is unique, so i starts at 1. trivial
3569         * edge cases first; no work needs to be done for either
3570         */
3571        if (length == 0 || length == 1)
3572                return length;
3573        /* src and dest walk down the list; dest counts unique elements */
3574        for (src = 1; src < length; src++) {
3575                /* find next unique element */
3576                while (list[src] == list[src-1]) {
3577                        src++;
3578                        if (src == length)
3579                                goto after;
3580                }
3581                /* dest always points to where the next unique element goes */
3582                list[dest] = list[src];
3583                dest++;
3584        }
3585after:
3586        return dest;
3587}
3588
3589static int cmppid(const void *a, const void *b)
3590{
3591        return *(pid_t *)a - *(pid_t *)b;
3592}
3593
3594/*
3595 * find the appropriate pidlist for our purpose (given procs vs tasks)
3596 * returns with the lock on that pidlist already held, and takes care
3597 * of the use count, or returns NULL with no locks held if we're out of
3598 * memory.
3599 */
3600static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
3601                                                  enum cgroup_filetype type)
3602{
3603        struct cgroup_pidlist *l;
3604        /* don't need task_nsproxy() if we're looking at ourself */
3605        struct pid_namespace *ns = task_active_pid_ns(current);
3606
3607        /*
3608         * We can't drop the pidlist_mutex before taking the l->rwsem in case
3609         * the last ref-holder is trying to remove l from the list at the same
3610         * time. Holding the pidlist_mutex precludes somebody taking whichever
3611         * list we find out from under us - compare release_pid_array().
3612         */
3613        mutex_lock(&cgrp->pidlist_mutex);
3614        list_for_each_entry(l, &cgrp->pidlists, links) {
3615                if (l->key.type == type && l->key.ns == ns) {
3616                        /* make sure l doesn't vanish out from under us */
3617                        down_write(&l->rwsem);
3618                        mutex_unlock(&cgrp->pidlist_mutex);
3619                        return l;
3620                }
3621        }
3622        /* entry not found; create a new one */
3623        l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
3624        if (!l) {
3625                mutex_unlock(&cgrp->pidlist_mutex);
3626                return l;
3627        }
3628        init_rwsem(&l->rwsem);
3629        down_write(&l->rwsem);
3630        l->key.type = type;
3631        l->key.ns = get_pid_ns(ns);
3632        l->owner = cgrp;
3633        list_add(&l->links, &cgrp->pidlists);
3634        mutex_unlock(&cgrp->pidlist_mutex);
3635        return l;
3636}
3637
3638/*
3639 * Load a cgroup's pidarray with either procs' tgids or tasks' pids
3640 */
3641static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
3642                              struct cgroup_pidlist **lp)
3643{
3644        pid_t *array;
3645        int length;
3646        int pid, n = 0; /* used for populating the array */
3647        struct css_task_iter it;
3648        struct task_struct *tsk;
3649        struct cgroup_pidlist *l;
3650
3651        /*
3652         * If cgroup gets more users after we read count, we won't have
3653         * enough space - tough.  This race is indistinguishable to the
3654         * caller from the case that the additional cgroup users didn't
3655         * show up until sometime later on.
3656         */
3657        length = cgroup_task_count(cgrp);
3658        array = pidlist_allocate(length);
3659        if (!array)
3660                return -ENOMEM;
3661        /* now, populate the array */
3662        css_task_iter_start(&cgrp->dummy_css, &it);
3663        while ((tsk = css_task_iter_next(&it))) {
3664                if (unlikely(n == length))
3665                        break;
3666                /* get tgid or pid for procs or tasks file respectively */
3667                if (type == CGROUP_FILE_PROCS)
3668                        pid = task_tgid_vnr(tsk);
3669                else
3670                        pid = task_pid_vnr(tsk);
3671                if (pid > 0) /* make sure to only use valid results */
3672                        array[n++] = pid;
3673        }
3674        css_task_iter_end(&it);
3675        length = n;
3676        /* now sort & (if procs) strip out duplicates */
3677        sort(array, length, sizeof(pid_t), cmppid, NULL);
3678        if (type == CGROUP_FILE_PROCS)
3679                length = pidlist_uniq(array, length);
3680        l = cgroup_pidlist_find(cgrp, type);
3681        if (!l) {
3682                pidlist_free(array);
3683                return -ENOMEM;
3684        }
3685        /* store array, freeing old if necessary - lock already held */
3686        pidlist_free(l->list);
3687        l->list = array;
3688        l->length = length;
3689        l->use_count++;
3690        up_write(&l->rwsem);
3691        *lp = l;
3692        return 0;
3693}
3694
3695/**
3696 * cgroupstats_build - build and fill cgroupstats
3697 * @stats: cgroupstats to fill information into
3698 * @dentry: A dentry entry belonging to the cgroup for which stats have
3699 * been requested.
3700 *
3701 * Build and fill cgroupstats so that taskstats can export it to user
3702 * space.
3703 */
3704int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
3705{
3706        int ret = -EINVAL;
3707        struct cgroup *cgrp;
3708        struct css_task_iter it;
3709        struct task_struct *tsk;
3710
3711        /*
3712         * Validate dentry by checking the superblock operations,
3713         * and make sure it's a directory.
3714         */
3715        if (dentry->d_sb->s_op != &cgroup_ops ||
3716            !S_ISDIR(dentry->d_inode->i_mode))
3717                 goto err;
3718
3719        ret = 0;
3720        cgrp = dentry->d_fsdata;
3721
3722        css_task_iter_start(&cgrp->dummy_css, &it);
3723        while ((tsk = css_task_iter_next(&it))) {
3724                switch (tsk->state) {
3725                case TASK_RUNNING:
3726                        stats->nr_running++;
3727                        break;
3728                case TASK_INTERRUPTIBLE:
3729                        stats->nr_sleeping++;
3730                        break;
3731                case TASK_UNINTERRUPTIBLE:
3732                        stats->nr_uninterruptible++;
3733                        break;
3734                case TASK_STOPPED:
3735                        stats->nr_stopped++;
3736                        break;
3737                default:
3738                        if (delayacct_is_task_waiting_on_io(tsk))
3739                                stats->nr_io_wait++;
3740                        break;
3741                }
3742        }
3743        css_task_iter_end(&it);
3744
3745err:
3746        return ret;
3747}
3748
3749
3750/*
3751 * seq_file methods for the tasks/procs files. The seq_file position is the
3752 * next pid to display; the seq_file iterator is a pointer to the pid
3753 * in the cgroup->l->list array.
3754 */
3755
3756static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
3757{
3758        /*
3759         * Initially we receive a position value that corresponds to
3760         * one more than the last pid shown (or 0 on the first call or
3761         * after a seek to the start). Use a binary-search to find the
3762         * next pid to display, if any
3763         */
3764        struct cgroup_pidlist *l = s->private;
3765        int index = 0, pid = *pos;
3766        int *iter;
3767
3768        down_read(&l->rwsem);
3769        if (pid) {
3770                int end = l->length;
3771
3772                while (index < end) {
3773                        int mid = (index + end) / 2;
3774                        if (l->list[mid] == pid) {
3775                                index = mid;
3776                                break;
3777                        } else if (l->list[mid] <= pid)
3778                                index = mid + 1;
3779                        else
3780                                end = mid;
3781                }
3782        }
3783        /* If we're off the end of the array, we're done */
3784        if (index >= l->length)
3785                return NULL;
3786        /* Update the abstract position to be the actual pid that we found */
3787        iter = l->list + index;
3788        *pos = *iter;
3789        return iter;
3790}
3791
3792static void cgroup_pidlist_stop(struct seq_file *s, void *v)
3793{
3794        struct cgroup_pidlist *l = s->private;
3795        up_read(&l->rwsem);
3796}
3797
3798static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
3799{
3800        struct cgroup_pidlist *l = s->private;
3801        pid_t *p = v;
3802        pid_t *end = l->list + l->length;
3803        /*
3804         * Advance to the next pid in the array. If this goes off the
3805         * end, we're done
3806         */
3807        p++;
3808        if (p >= end) {
3809                return NULL;
3810        } else {
3811                *pos = *p;
3812                return p;
3813        }
3814}
3815
3816static int cgroup_pidlist_show(struct seq_file *s, void *v)
3817{
3818        return seq_printf(s, "%d\n", *(int *)v);
3819}
3820
3821/*
3822 * seq_operations functions for iterating on pidlists through seq_file -
3823 * independent of whether it's tasks or procs
3824 */
3825static const struct seq_operations cgroup_pidlist_seq_operations = {
3826        .start = cgroup_pidlist_start,
3827        .stop = cgroup_pidlist_stop,
3828        .next = cgroup_pidlist_next,
3829        .show = cgroup_pidlist_show,
3830};
3831
3832static void cgroup_release_pid_array(struct cgroup_pidlist *l)
3833{
3834        /*
3835         * the case where we're the last user of this particular pidlist will
3836         * have us remove it from the cgroup's list, which entails taking the
3837         * mutex. since in pidlist_find the pidlist->lock depends on cgroup->
3838         * pidlist_mutex, we have to take pidlist_mutex first.
3839         */
3840        mutex_lock(&l->owner->pidlist_mutex);
3841        down_write(&l->rwsem);
3842        BUG_ON(!l->use_count);
3843        if (!--l->use_count) {
3844                /* we're the last user if refcount is 0; remove and free */
3845                list_del(&l->links);
3846                mutex_unlock(&l->owner->pidlist_mutex);
3847                pidlist_free(l->list);
3848                put_pid_ns(l->key.ns);
3849                up_write(&l->rwsem);
3850                kfree(l);
3851                return;
3852        }
3853        mutex_unlock(&l->owner->pidlist_mutex);
3854        up_write(&l->rwsem);
3855}
3856
3857static int cgroup_pidlist_release(struct inode *inode, struct file *file)
3858{
3859        struct cgroup_pidlist *l;
3860        if (!(file->f_mode & FMODE_READ))
3861                return 0;
3862        /*
3863         * the seq_file will only be initialized if the file was opened for
3864         * reading; hence we check if it's not null only in that case.
3865         */
3866        l = ((struct seq_file *)file->private_data)->private;
3867        cgroup_release_pid_array(l);
3868        return seq_release(inode, file);
3869}
3870
3871static const struct file_operations cgroup_pidlist_operations = {
3872        .read = seq_read,
3873        .llseek = seq_lseek,
3874        .write = cgroup_file_write,
3875        .release = cgroup_pidlist_release,
3876};
3877
3878/*
3879 * The following functions handle opens on a file that displays a pidlist
3880 * (tasks or procs). Prepare an array of the process/thread IDs of whoever's
3881 * in the cgroup.
3882 */
3883/* helper function for the two below it */
3884static int cgroup_pidlist_open(struct file *file, enum cgroup_filetype type)
3885{
3886        struct cgroup *cgrp = __d_cgrp(file->f_dentry->d_parent);
3887        struct cgroup_pidlist *l;
3888        int retval;
3889
3890        /* Nothing to do for write-only files */
3891        if (!(file->f_mode & FMODE_READ))
3892                return 0;
3893
3894        /* have the array populated */
3895        retval = pidlist_array_load(cgrp, type, &l);
3896        if (retval)
3897                return retval;
3898        /* configure file information */
3899        file->f_op = &cgroup_pidlist_operations;
3900
3901        retval = seq_open(file, &cgroup_pidlist_seq_operations);
3902        if (retval) {
3903                cgroup_release_pid_array(l);
3904                return retval;
3905        }
3906        ((struct seq_file *)file->private_data)->private = l;
3907        return 0;
3908}
3909static int cgroup_tasks_open(struct inode *unused, struct file *file)
3910{
3911        return cgroup_pidlist_open(file, CGROUP_FILE_TASKS);
3912}
3913static int cgroup_procs_open(struct inode *unused, struct file *file)
3914{
3915        return cgroup_pidlist_open(file, CGROUP_FILE_PROCS);
3916}
3917
3918static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
3919                                         struct cftype *cft)
3920{
3921        return notify_on_release(css->cgroup);
3922}
3923
3924static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
3925                                          struct cftype *cft, u64 val)
3926{
3927        clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
3928        if (val)
3929                set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
3930        else
3931                clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
3932        return 0;
3933}
3934
3935/*
3936 * When dput() is called asynchronously, if umount has been done and
3937 * then deactivate_super() in cgroup_free_fn() kills the superblock,
3938 * there's a small window that vfs will see the root dentry with non-zero
3939 * refcnt and trigger BUG().
3940 *
3941 * That's why we hold a reference before dput() and drop it right after.
3942 */
3943static void cgroup_dput(struct cgroup *cgrp)
3944{
3945        struct super_block *sb = cgrp->root->sb;
3946
3947        atomic_inc(&sb->s_active);
3948        dput(cgrp->dentry);
3949        deactivate_super(sb);
3950}
3951
3952/*
3953 * Unregister event and free resources.
3954 *
3955 * Gets called from workqueue.
3956 */
3957static void cgroup_event_remove(struct work_struct *work)
3958{
3959        struct cgroup_event *event = container_of(work, struct cgroup_event,
3960                        remove);
3961        struct cgroup_subsys_state *css = event->css;
3962
3963        remove_wait_queue(event->wqh, &event->wait);
3964
3965        event->cft->unregister_event(css, event->cft, event->eventfd);
3966
3967        /* Notify userspace the event is going away. */
3968        eventfd_signal(event->eventfd, 1);
3969
3970        eventfd_ctx_put(event->eventfd);
3971        kfree(event);
3972        css_put(css);
3973}
3974
3975/*
3976 * Gets called on POLLHUP on eventfd when user closes it.
3977 *
3978 * Called with wqh->lock held and interrupts disabled.
3979 */
3980static int cgroup_event_wake(wait_queue_t *wait, unsigned mode,
3981                int sync, void *key)
3982{
3983        struct cgroup_event *event = container_of(wait,
3984                        struct cgroup_event, wait);
3985        struct cgroup *cgrp = event->css->cgroup;
3986        unsigned long flags = (unsigned long)key;
3987
3988        if (flags & POLLHUP) {
3989                /*
3990                 * If the event has been detached at cgroup removal, we
3991                 * can simply return knowing the other side will cleanup
3992                 * for us.
3993                 *
3994                 * We can't race against event freeing since the other
3995                 * side will require wqh->lock via remove_wait_queue(),
3996                 * which we hold.
3997                 */
3998                spin_lock(&cgrp->event_list_lock);
3999                if (!list_empty(&event->list)) {
4000                        list_del_init(&event->list);
4001                        /*
4002                         * We are in atomic context, but cgroup_event_remove()
4003                         * may sleep, so we have to call it in workqueue.
4004                         */
4005                        schedule_work(&event->remove);
4006                }
4007                spin_unlock(&cgrp->event_list_lock);
4008        }
4009
4010        return 0;
4011}
4012
4013static void cgroup_event_ptable_queue_proc(struct file *file,
4014                wait_queue_head_t *wqh, poll_table *pt)
4015{
4016        struct cgroup_event *event = container_of(pt,
4017                        struct cgroup_event, pt);
4018
4019        event->wqh = wqh;
4020        add_wait_queue(wqh, &event->wait);
4021}
4022
4023/*
4024 * Parse input and register new cgroup event handler.
4025 *
4026 * Input must be in format '<event_fd> <control_fd> <args>'.
4027 * Interpretation of args is defined by control file implementation.
4028 */
4029static int cgroup_write_event_control(struct cgroup_subsys_state *dummy_css,
4030                                      struct cftype *cft, const char *buffer)
4031{
4032        struct cgroup *cgrp = dummy_css->cgroup;
4033        struct cgroup_event *event;
4034        struct cgroup_subsys_state *cfile_css;
4035        unsigned int efd, cfd;
4036        struct fd efile;
4037        struct fd cfile;
4038        char *endp;
4039        int ret;
4040
4041        efd = simple_strtoul(buffer, &endp, 10);
4042        if (*endp != ' ')
4043                return -EINVAL;
4044        buffer = endp + 1;
4045
4046        cfd = simple_strtoul(buffer, &endp, 10);
4047        if ((*endp != ' ') && (*endp != '\0'))
4048                return -EINVAL;
4049        buffer = endp + 1;
4050
4051        event = kzalloc(sizeof(*event), GFP_KERNEL);
4052        if (!event)
4053                return -ENOMEM;
4054
4055        INIT_LIST_HEAD(&event->list);
4056        init_poll_funcptr(&event->pt, cgroup_event_ptable_queue_proc);
4057        init_waitqueue_func_entry(&event->wait, cgroup_event_wake);
4058        INIT_WORK(&event->remove, cgroup_event_remove);
4059
4060        efile = fdget(efd);
4061        if (!efile.file) {
4062                ret = -EBADF;
4063                goto out_kfree;
4064        }
4065
4066        event->eventfd = eventfd_ctx_fileget(efile.file);
4067        if (IS_ERR(event->eventfd)) {
4068                ret = PTR_ERR(event->eventfd);
4069                goto out_put_efile;
4070        }
4071
4072        cfile = fdget(cfd);
4073        if (!cfile.file) {
4074                ret = -EBADF;
4075                goto out_put_eventfd;
4076        }
4077
4078        /* the process need read permission on control file */
4079        /* AV: shouldn't we check that it's been opened for read instead? */
4080        ret = inode_permission(file_inode(cfile.file), MAY_READ);
4081        if (ret < 0)
4082                goto out_put_cfile;
4083
4084        event->cft = __file_cft(cfile.file);
4085        if (IS_ERR(event->cft)) {
4086                ret = PTR_ERR(event->cft);
4087                goto out_put_cfile;
4088        }
4089
4090        if (!event->cft->ss) {
4091                ret = -EBADF;
4092                goto out_put_cfile;
4093        }
4094
4095        /*
4096         * Determine the css of @cfile, verify it belongs to the same
4097         * cgroup as cgroup.event_control, and associate @event with it.
4098         * Remaining events are automatically removed on cgroup destruction
4099         * but the removal is asynchronous, so take an extra ref.
4100         */
4101        rcu_read_lock();
4102
4103        ret = -EINVAL;
4104        event->css = cgroup_css(cgrp, event->cft->ss);
4105        cfile_css = css_from_dir(cfile.file->f_dentry->d_parent, event->cft->ss);
4106        if (event->css && event->css == cfile_css && css_tryget(event->css))
4107                ret = 0;
4108
4109        rcu_read_unlock();
4110        if (ret)
4111                goto out_put_cfile;
4112
4113        if (!event->cft->register_event || !event->cft->unregister_event) {
4114                ret = -EINVAL;
4115                goto out_put_css;
4116        }
4117
4118        ret = event->cft->register_event(event->css, event->cft,
4119                        event->eventfd, buffer);
4120        if (ret)
4121                goto out_put_css;
4122
4123        efile.file->f_op->poll(efile.file, &event->pt);
4124
4125        spin_lock(&cgrp->event_list_lock);
4126        list_add(&event->list, &cgrp->event_list);
4127        spin_unlock(&cgrp->event_list_lock);
4128
4129        fdput(cfile);
4130        fdput(efile);
4131
4132        return 0;
4133
4134out_put_css:
4135        css_put(event->css);
4136out_put_cfile:
4137        fdput(cfile);
4138out_put_eventfd:
4139        eventfd_ctx_put(event->eventfd);
4140out_put_efile:
4141        fdput(efile);
4142out_kfree:
4143        kfree(event);
4144
4145        return ret;
4146}
4147
4148static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
4149                                      struct cftype *cft)
4150{
4151        return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4152}
4153
4154static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
4155                                       struct cftype *cft, u64 val)
4156{
4157        if (val)
4158                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4159        else
4160                clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4161        return 0;
4162}
4163
4164static struct cftype cgroup_base_files[] = {
4165        {
4166                .name = "cgroup.procs",
4167                .open = cgroup_procs_open,
4168                .write_u64 = cgroup_procs_write,
4169                .release = cgroup_pidlist_release,
4170                .mode = S_IRUGO | S_IWUSR,
4171        },
4172        {
4173                .name = "cgroup.event_control",
4174                .write_string = cgroup_write_event_control,
4175                .mode = S_IWUGO,
4176        },
4177        {
4178                .name = "cgroup.clone_children",
4179                .flags = CFTYPE_INSANE,
4180                .read_u64 = cgroup_clone_children_read,
4181                .write_u64 = cgroup_clone_children_write,
4182        },
4183        {
4184                .name = "cgroup.sane_behavior",
4185                .flags = CFTYPE_ONLY_ON_ROOT,
4186                .read_seq_string = cgroup_sane_behavior_show,
4187        },
4188
4189        /*
4190         * Historical crazy stuff.  These don't have "cgroup."  prefix and
4191         * don't exist if sane_behavior.  If you're depending on these, be
4192         * prepared to be burned.
4193         */
4194        {
4195                .name = "tasks",
4196                .flags = CFTYPE_INSANE,         /* use "procs" instead */
4197                .open = cgroup_tasks_open,
4198                .write_u64 = cgroup_tasks_write,
4199                .release = cgroup_pidlist_release,
4200                .mode = S_IRUGO | S_IWUSR,
4201        },
4202        {
4203                .name = "notify_on_release",
4204                .flags = CFTYPE_INSANE,
4205                .read_u64 = cgroup_read_notify_on_release,
4206                .write_u64 = cgroup_write_notify_on_release,
4207        },
4208        {
4209                .name = "release_agent",
4210                .flags = CFTYPE_INSANE | CFTYPE_ONLY_ON_ROOT,
4211                .read_seq_string = cgroup_release_agent_show,
4212                .write_string = cgroup_release_agent_write,
4213                .max_write_len = PATH_MAX,
4214        },
4215        { }     /* terminate */
4216};
4217
4218/**
4219 * cgroup_populate_dir - create subsys files in a cgroup directory
4220 * @cgrp: target cgroup
4221 * @subsys_mask: mask of the subsystem ids whose files should be added
4222 *
4223 * On failure, no file is added.
4224 */
4225static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask)
4226{
4227        struct cgroup_subsys *ss;
4228        int i, ret = 0;
4229
4230        /* process cftsets of each subsystem */
4231        for_each_subsys(ss, i) {
4232                struct cftype_set *set;
4233
4234                if (!test_bit(i, &subsys_mask))
4235                        continue;
4236
4237                list_for_each_entry(set, &ss->cftsets, node) {
4238                        ret = cgroup_addrm_files(cgrp, set->cfts, true);
4239                        if (ret < 0)
4240                                goto err;
4241                }
4242        }
4243
4244        /* This cgroup is ready now */
4245        for_each_root_subsys(cgrp->root, ss) {
4246                struct cgroup_subsys_state *css = cgroup_css(cgrp, ss);
4247                struct css_id *id = rcu_dereference_protected(css->id, true);
4248
4249                /*
4250                 * Update id->css pointer and make this css visible from
4251                 * CSS ID functions. This pointer will be dereferened
4252                 * from RCU-read-side without locks.
4253                 */
4254                if (id)
4255                        rcu_assign_pointer(id->css, css);
4256        }
4257
4258        return 0;
4259err:
4260        cgroup_clear_dir(cgrp, subsys_mask);
4261        return ret;
4262}
4263
4264/*
4265 * css destruction is four-stage process.
4266 *
4267 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
4268 *    Implemented in kill_css().
4269 *
4270 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4271 *    and thus css_tryget() is guaranteed to fail, the css can be offlined
4272 *    by invoking offline_css().  After offlining, the base ref is put.
4273 *    Implemented in css_killed_work_fn().
4274 *
4275 * 3. When the percpu_ref reaches zero, the only possible remaining
4276 *    accessors are inside RCU read sections.  css_release() schedules the
4277 *    RCU callback.
4278 *
4279 * 4. After the grace period, the css can be freed.  Implemented in
4280 *    css_free_work_fn().
4281 *
4282 * It is actually hairier because both step 2 and 4 require process context
4283 * and thus involve punting to css->destroy_work adding two additional
4284 * steps to the already complex sequence.
4285 */
4286static void css_free_work_fn(struct work_struct *work)
4287{
4288        struct cgroup_subsys_state *css =
4289                container_of(work, struct cgroup_subsys_state, destroy_work);
4290        struct cgroup *cgrp = css->cgroup;
4291
4292        if (css->parent)
4293                css_put(css->parent);
4294
4295        css->ss->css_free(css);
4296        cgroup_dput(cgrp);
4297}
4298
4299static void css_free_rcu_fn(struct rcu_head *rcu_head)
4300{
4301        struct cgroup_subsys_state *css =
4302                container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4303
4304        /*
4305         * css holds an extra ref to @cgrp->dentry which is put on the last
4306         * css_put().  dput() requires process context which we don't have.
4307         */
4308        INIT_WORK(&css->destroy_work, css_free_work_fn);
4309        schedule_work(&css->destroy_work);
4310}
4311
4312static void css_release(struct percpu_ref *ref)
4313{
4314        struct cgroup_subsys_state *css =
4315                container_of(ref, struct cgroup_subsys_state, refcnt);
4316
4317        call_rcu(&css->rcu_head, css_free_rcu_fn);
4318}
4319
4320static void init_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss,
4321                     struct cgroup *cgrp)
4322{
4323        css->cgroup = cgrp;
4324        css->ss = ss;
4325        css->flags = 0;
4326        css->id = NULL;
4327
4328        if (cgrp->parent)
4329                css->parent = cgroup_css(cgrp->parent, ss);
4330        else
4331                css->flags |= CSS_ROOT;
4332
4333        BUG_ON(cgroup_css(cgrp, ss));
4334}
4335
4336/* invoke ->css_online() on a new CSS and mark it online if successful */
4337static int online_css(struct cgroup_subsys_state *css)
4338{
4339        struct cgroup_subsys *ss = css->ss;
4340        int ret = 0;
4341
4342        lockdep_assert_held(&cgroup_mutex);
4343
4344        if (ss->css_online)
4345                ret = ss->css_online(css);
4346        if (!ret) {
4347                css->flags |= CSS_ONLINE;
4348                css->cgroup->nr_css++;
4349                rcu_assign_pointer(css->cgroup->subsys[ss->subsys_id], css);
4350        }
4351        return ret;
4352}
4353
4354/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4355static void offline_css(struct cgroup_subsys_state *css)
4356{
4357        struct cgroup_subsys *ss = css->ss;
4358
4359        lockdep_assert_held(&cgroup_mutex);
4360
4361        if (!(css->flags & CSS_ONLINE))
4362                return;
4363
4364        if (ss->css_offline)
4365                ss->css_offline(css);
4366
4367        css->flags &= ~CSS_ONLINE;
4368        css->cgroup->nr_css--;
4369        RCU_INIT_POINTER(css->cgroup->subsys[ss->subsys_id], css);
4370}
4371
4372/*
4373 * cgroup_create - create a cgroup
4374 * @parent: cgroup that will be parent of the new cgroup
4375 * @dentry: dentry of the new cgroup
4376 * @mode: mode to set on new inode
4377 *
4378 * Must be called with the mutex on the parent inode held
4379 */
4380static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
4381                             umode_t mode)
4382{
4383        struct cgroup_subsys_state *css_ar[CGROUP_SUBSYS_COUNT] = { };
4384        struct cgroup *cgrp;
4385        struct cgroup_name *name;
4386        struct cgroupfs_root *root = parent->root;
4387        int err = 0;
4388        struct cgroup_subsys *ss;
4389        struct super_block *sb = root->sb;
4390
4391        /* allocate the cgroup and its ID, 0 is reserved for the root */
4392        cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
4393        if (!cgrp)
4394                return -ENOMEM;
4395
4396        name = cgroup_alloc_name(dentry);
4397        if (!name)
4398                goto err_free_cgrp;
4399        rcu_assign_pointer(cgrp->name, name);
4400
4401        /*
4402         * Temporarily set the pointer to NULL, so idr_find() won't return
4403         * a half-baked cgroup.
4404         */
4405        cgrp->id = idr_alloc(&root->cgroup_idr, NULL, 1, 0, GFP_KERNEL);
4406        if (cgrp->id < 0)
4407                goto err_free_name;
4408
4409        /*
4410         * Only live parents can have children.  Note that the liveliness
4411         * check isn't strictly necessary because cgroup_mkdir() and
4412         * cgroup_rmdir() are fully synchronized by i_mutex; however, do it
4413         * anyway so that locking is contained inside cgroup proper and we
4414         * don't get nasty surprises if we ever grow another caller.
4415         */
4416        if (!cgroup_lock_live_group(parent)) {
4417                err = -ENODEV;
4418                goto err_free_id;
4419        }
4420
4421        /* Grab a reference on the superblock so the hierarchy doesn't
4422         * get deleted on unmount if there are child cgroups.  This
4423         * can be done outside cgroup_mutex, since the sb can't
4424         * disappear while someone has an open control file on the
4425         * fs */
4426        atomic_inc(&sb->s_active);
4427
4428        init_cgroup_housekeeping(cgrp);
4429
4430        dentry->d_fsdata = cgrp;
4431        cgrp->dentry = dentry;
4432
4433        cgrp->parent = parent;
4434        cgrp->dummy_css.parent = &parent->dummy_css;
4435        cgrp->root = parent->root;
4436
4437        if (notify_on_release(parent))
4438                set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
4439
4440        if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
4441                set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4442
4443        for_each_root_subsys(root, ss) {
4444                struct cgroup_subsys_state *css;
4445
4446                css = ss->css_alloc(cgroup_css(parent, ss));
4447                if (IS_ERR(css)) {
4448                        err = PTR_ERR(css);
4449                        goto err_free_all;
4450                }
4451                css_ar[ss->subsys_id] = css;
4452
4453                err = percpu_ref_init(&css->refcnt, css_release);
4454                if (err)
4455                        goto err_free_all;
4456
4457                init_css(css, ss, cgrp);
4458
4459                if (ss->use_id) {
4460                        err = alloc_css_id(css);
4461                        if (err)
4462                                goto err_free_all;
4463                }
4464        }
4465
4466        /*
4467         * Create directory.  cgroup_create_file() returns with the new
4468         * directory locked on success so that it can be populated without
4469         * dropping cgroup_mutex.
4470         */
4471        err = cgroup_create_file(dentry, S_IFDIR | mode, sb);
4472        if (err < 0)
4473                goto err_free_all;
4474        lockdep_assert_held(&dentry->d_inode->i_mutex);
4475
4476        cgrp->serial_nr = cgroup_serial_nr_next++;
4477
4478        /* allocation complete, commit to creation */
4479        list_add_tail_rcu(&cgrp->sibling, &cgrp->parent->children);
4480        root->number_of_cgroups++;
4481
4482        /* each css holds a ref to the cgroup's dentry and the parent css */
4483        for_each_root_subsys(root, ss) {
4484                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
4485
4486                dget(dentry);
4487                css_get(css->parent);
4488        }
4489
4490        /* hold a ref to the parent's dentry */
4491        dget(parent->dentry);
4492
4493        /* creation succeeded, notify subsystems */
4494        for_each_root_subsys(root, ss) {
4495                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
4496
4497                err = online_css(css);
4498                if (err)
4499                        goto err_destroy;
4500
4501                if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
4502                    parent->parent) {
4503                        pr_warning("cgroup: %s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4504                                   current->comm, current->pid, ss->name);
4505                        if (!strcmp(ss->name, "memory"))
4506                                pr_warning("cgroup: \"memory\" requires setting use_hierarchy to 1 on the root.\n");
4507                        ss->warned_broken_hierarchy = true;
4508                }
4509        }
4510
4511        idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4512
4513        err = cgroup_addrm_files(cgrp, cgroup_base_files, true);
4514        if (err)
4515                goto err_destroy;
4516
4517        err = cgroup_populate_dir(cgrp, root->subsys_mask);
4518        if (err)
4519                goto err_destroy;
4520
4521        mutex_unlock(&cgroup_mutex);
4522        mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
4523
4524        return 0;
4525
4526err_free_all:
4527        for_each_root_subsys(root, ss) {
4528                struct cgroup_subsys_state *css = css_ar[ss->subsys_id];
4529
4530                if (css) {
4531                        percpu_ref_cancel_init(&css->refcnt);
4532                        ss->css_free(css);
4533                }
4534        }
4535        mutex_unlock(&cgroup_mutex);
4536        /* Release the reference count that we took on the superblock */
4537        deactivate_super(sb);
4538err_free_id:
4539        idr_remove(&root->cgroup_idr, cgrp->id);
4540err_free_name:
4541        kfree(rcu_dereference_raw(cgrp->name));
4542err_free_cgrp:
4543        kfree(cgrp);
4544        return err;
4545
4546err_destroy:
4547        cgroup_destroy_locked(cgrp);
4548        mutex_unlock(&cgroup_mutex);
4549        mutex_unlock(&dentry->d_inode->i_mutex);
4550        return err;
4551}
4552
4553static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
4554{
4555        struct cgroup *c_parent = dentry->d_parent->d_fsdata;
4556
4557        /* the vfs holds inode->i_mutex already */
4558        return cgroup_create(c_parent, dentry, mode | S_IFDIR);
4559}
4560
4561/*
4562 * This is called when the refcnt of a css is confirmed to be killed.
4563 * css_tryget() is now guaranteed to fail.
4564 */
4565static void css_killed_work_fn(struct work_struct *work)
4566{
4567        struct cgroup_subsys_state *css =
4568                container_of(work, struct cgroup_subsys_state, destroy_work);
4569        struct cgroup *cgrp = css->cgroup;
4570
4571        mutex_lock(&cgroup_mutex);
4572
4573        /*
4574         * css_tryget() is guaranteed to fail now.  Tell subsystems to
4575         * initate destruction.
4576         */
4577        offline_css(css);
4578
4579        /*
4580         * If @cgrp is marked dead, it's waiting for refs of all css's to
4581         * be disabled before proceeding to the second phase of cgroup
4582         * destruction.  If we are the last one, kick it off.
4583         */
4584        if (!cgrp->nr_css && cgroup_is_dead(cgrp))
4585                cgroup_destroy_css_killed(cgrp);
4586
4587        mutex_unlock(&cgroup_mutex);
4588
4589        /*
4590         * Put the css refs from kill_css().  Each css holds an extra
4591         * reference to the cgroup's dentry and cgroup removal proceeds
4592         * regardless of css refs.  On the last put of each css, whenever
4593         * that may be, the extra dentry ref is put so that dentry
4594         * destruction happens only after all css's are released.
4595         */
4596        css_put(css);
4597}
4598
4599/* css kill confirmation processing requires process context, bounce */
4600static void css_killed_ref_fn(struct percpu_ref *ref)
4601{
4602        struct cgroup_subsys_state *css =
4603                container_of(ref, struct cgroup_subsys_state, refcnt);
4604
4605        INIT_WORK(&css->destroy_work, css_killed_work_fn);
4606        schedule_work(&css->destroy_work);
4607}
4608
4609/**
4610 * kill_css - destroy a css
4611 * @css: css to destroy
4612 *
4613 * This function initiates destruction of @css by removing cgroup interface
4614 * files and putting its base reference.  ->css_offline() will be invoked
4615 * asynchronously once css_tryget() is guaranteed to fail and when the
4616 * reference count reaches zero, @css will be released.
4617 */
4618static void kill_css(struct cgroup_subsys_state *css)
4619{
4620        cgroup_clear_dir(css->cgroup, 1 << css->ss->subsys_id);
4621
4622        /*
4623         * Killing would put the base ref, but we need to keep it alive
4624         * until after ->css_offline().
4625         */
4626        css_get(css);
4627
4628        /*
4629         * cgroup core guarantees that, by the time ->css_offline() is
4630         * invoked, no new css reference will be given out via
4631         * css_tryget().  We can't simply call percpu_ref_kill() and
4632         * proceed to offlining css's because percpu_ref_kill() doesn't
4633         * guarantee that the ref is seen as killed on all CPUs on return.
4634         *
4635         * Use percpu_ref_kill_and_confirm() to get notifications as each
4636         * css is confirmed to be seen as killed on all CPUs.
4637         */
4638        percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4639}
4640
4641/**
4642 * cgroup_destroy_locked - the first stage of cgroup destruction
4643 * @cgrp: cgroup to be destroyed
4644 *
4645 * css's make use of percpu refcnts whose killing latency shouldn't be
4646 * exposed to userland and are RCU protected.  Also, cgroup core needs to
4647 * guarantee that css_tryget() won't succeed by the time ->css_offline() is
4648 * invoked.  To satisfy all the requirements, destruction is implemented in
4649 * the following two steps.
4650 *
4651 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
4652 *     userland visible parts and start killing the percpu refcnts of
4653 *     css's.  Set up so that the next stage will be kicked off once all
4654 *     the percpu refcnts are confirmed to be killed.
4655 *
4656 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
4657 *     rest of destruction.  Once all cgroup references are gone, the
4658 *     cgroup is RCU-freed.
4659 *
4660 * This function implements s1.  After this step, @cgrp is gone as far as
4661 * the userland is concerned and a new cgroup with the same name may be
4662 * created.  As cgroup doesn't care about the names internally, this
4663 * doesn't cause any problem.
4664 */
4665static int cgroup_destroy_locked(struct cgroup *cgrp)
4666        __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4667{
4668        struct dentry *d = cgrp->dentry;
4669        struct cgroup_event *event, *tmp;
4670        struct cgroup_subsys *ss;
4671        struct cgroup *child;
4672        bool empty;
4673
4674        lockdep_assert_held(&d->d_inode->i_mutex);
4675        lockdep_assert_held(&cgroup_mutex);
4676
4677        /*
4678         * css_set_lock synchronizes access to ->cset_links and prevents
4679         * @cgrp from being removed while __put_css_set() is in progress.
4680         */
4681        read_lock(&css_set_lock);
4682        empty = list_empty(&cgrp->cset_links);
4683        read_unlock(&css_set_lock);
4684        if (!empty)
4685                return -EBUSY;
4686
4687        /*
4688         * Make sure there's no live children.  We can't test ->children
4689         * emptiness as dead children linger on it while being destroyed;
4690         * otherwise, "rmdir parent/child parent" may fail with -EBUSY.
4691         */
4692        empty = true;
4693        rcu_read_lock();
4694        list_for_each_entry_rcu(child, &cgrp->children, sibling) {
4695                empty = cgroup_is_dead(child);
4696                if (!empty)
4697                        break;
4698        }
4699        rcu_read_unlock();
4700        if (!empty)
4701                return -EBUSY;
4702
4703        /*
4704         * Initiate massacre of all css's.  cgroup_destroy_css_killed()
4705         * will be invoked to perform the rest of destruction once the
4706         * percpu refs of all css's are confirmed to be killed.
4707         */
4708        for_each_root_subsys(cgrp->root, ss)
4709                kill_css(cgroup_css(cgrp, ss));
4710
4711        /*
4712         * Mark @cgrp dead.  This prevents further task migration and child
4713         * creation by disabling cgroup_lock_live_group().  Note that
4714         * CGRP_DEAD assertion is depended upon by css_next_child() to
4715         * resume iteration after dropping RCU read lock.  See
4716         * css_next_child() for details.
4717         */
4718        set_bit(CGRP_DEAD, &cgrp->flags);
4719
4720        /* CGRP_DEAD is set, remove from ->release_list for the last time */
4721        raw_spin_lock(&release_list_lock);
4722        if (!list_empty(&cgrp->release_list))
4723                list_del_init(&cgrp->release_list);
4724        raw_spin_unlock(&release_list_lock);
4725
4726        /*
4727         * If @cgrp has css's attached, the second stage of cgroup
4728         * destruction is kicked off from css_killed_work_fn() after the
4729         * refs of all attached css's are killed.  If @cgrp doesn't have
4730         * any css, we kick it off here.
4731         */
4732        if (!cgrp->nr_css)
4733                cgroup_destroy_css_killed(cgrp);
4734
4735        /*
4736         * Clear the base files and remove @cgrp directory.  The removal
4737         * puts the base ref but we aren't quite done with @cgrp yet, so
4738         * hold onto it.
4739         */
4740        cgroup_addrm_files(cgrp, cgroup_base_files, false);
4741        dget(d);
4742        cgroup_d_remove_dir(d);
4743
4744        /*
4745         * Unregister events and notify userspace.
4746         * Notify userspace about cgroup removing only after rmdir of cgroup
4747         * directory to avoid race between userspace and kernelspace.
4748         */
4749        spin_lock(&cgrp->event_list_lock);
4750        list_for_each_entry_safe(event, tmp, &cgrp->event_list, list) {
4751                list_del_init(&event->list);
4752                schedule_work(&event->remove);
4753        }
4754        spin_unlock(&cgrp->event_list_lock);
4755
4756        return 0;
4757};
4758
4759/**
4760 * cgroup_destroy_css_killed - the second step of cgroup destruction
4761 * @work: cgroup->destroy_free_work
4762 *
4763 * This function is invoked from a work item for a cgroup which is being
4764 * destroyed after all css's are offlined and performs the rest of
4765 * destruction.  This is the second step of destruction described in the
4766 * comment above cgroup_destroy_locked().
4767 */
4768static void cgroup_destroy_css_killed(struct cgroup *cgrp)
4769{
4770        struct cgroup *parent = cgrp->parent;
4771        struct dentry *d = cgrp->dentry;
4772
4773        lockdep_assert_held(&cgroup_mutex);
4774
4775        /* delete this cgroup from parent->children */
4776        list_del_rcu(&cgrp->sibling);
4777
4778        /*
4779         * We should remove the cgroup object from idr before its grace
4780         * period starts, so we won't be looking up a cgroup while the
4781         * cgroup is being freed.
4782         */
4783        idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
4784        cgrp->id = -1;
4785
4786        dput(d);
4787
4788        set_bit(CGRP_RELEASABLE, &parent->flags);
4789        check_for_release(parent);
4790}
4791
4792static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry)
4793{
4794        int ret;
4795
4796        mutex_lock(&cgroup_mutex);
4797        ret = cgroup_destroy_locked(dentry->d_fsdata);
4798        mutex_unlock(&cgroup_mutex);
4799
4800        return ret;
4801}
4802
4803static void __init_or_module cgroup_init_cftsets(struct cgroup_subsys *ss)
4804{
4805        INIT_LIST_HEAD(&ss->cftsets);
4806
4807        /*
4808         * base_cftset is embedded in subsys itself, no need to worry about
4809         * deregistration.
4810         */
4811        if (ss->base_cftypes) {
4812                struct cftype *cft;
4813
4814                for (cft = ss->base_cftypes; cft->name[0] != '\0'; cft++)
4815                        cft->ss = ss;
4816
4817                ss->base_cftset.cfts = ss->base_cftypes;
4818                list_add_tail(&ss->base_cftset.node, &ss->cftsets);
4819        }
4820}
4821
4822static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
4823{
4824        struct cgroup_subsys_state *css;
4825
4826        printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4827
4828        mutex_lock(&cgroup_mutex);
4829
4830        /* init base cftset */
4831        cgroup_init_cftsets(ss);
4832
4833        /* Create the top cgroup state for this subsystem */
4834        list_add(&ss->sibling, &cgroup_dummy_root.subsys_list);
4835        ss->root = &cgroup_dummy_root;
4836        css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
4837        /* We don't handle early failures gracefully */
4838        BUG_ON(IS_ERR(css));
4839        init_css(css, ss, cgroup_dummy_top);
4840
4841        /* Update the init_css_set to contain a subsys
4842         * pointer to this state - since the subsystem is
4843         * newly registered, all tasks and hence the
4844         * init_css_set is in the subsystem's top cgroup. */
4845        init_css_set.subsys[ss->subsys_id] = css;
4846
4847        need_forkexit_callback |= ss->fork || ss->exit;
4848
4849        /* At system boot, before all subsystems have been
4850         * registered, no tasks have been forked, so we don't
4851         * need to invoke fork callbacks here. */
4852        BUG_ON(!list_empty(&init_task.tasks));
4853
4854        BUG_ON(online_css(css));
4855
4856        mutex_unlock(&cgroup_mutex);
4857
4858        /* this function shouldn't be used with modular subsystems, since they
4859         * need to register a subsys_id, among other things */
4860        BUG_ON(ss->module);
4861}
4862
4863/**
4864 * cgroup_load_subsys: load and register a modular subsystem at runtime
4865 * @ss: the subsystem to load
4866 *
4867 * This function should be called in a modular subsystem's initcall. If the
4868 * subsystem is built as a module, it will be assigned a new subsys_id and set
4869 * up for use. If the subsystem is built-in anyway, work is delegated to the
4870 * simpler cgroup_init_subsys.
4871 */
4872int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
4873{
4874        struct cgroup_subsys_state *css;
4875        int i, ret;
4876        struct hlist_node *tmp;
4877        struct css_set *cset;
4878        unsigned long key;
4879
4880        /* check name and function validity */
4881        if (ss->name == NULL || strlen(ss->name) > MAX_CGROUP_TYPE_NAMELEN ||
4882            ss->css_alloc == NULL || ss->css_free == NULL)
4883                return -EINVAL;
4884
4885        /*
4886         * we don't support callbacks in modular subsystems. this check is
4887         * before the ss->module check for consistency; a subsystem that could
4888         * be a module should still have no callbacks even if the user isn't
4889         * compiling it as one.
4890         */
4891        if (ss->fork || ss->exit)
4892                return -EINVAL;
4893
4894        /*
4895         * an optionally modular subsystem is built-in: we want to do nothing,
4896         * since cgroup_init_subsys will have already taken care of it.
4897         */
4898        if (ss->module == NULL) {
4899                /* a sanity check */
4900                BUG_ON(cgroup_subsys[ss->subsys_id] != ss);
4901                return 0;
4902        }
4903
4904        /* init base cftset */
4905        cgroup_init_cftsets(ss);
4906
4907        mutex_lock(&cgroup_mutex);
4908        cgroup_subsys[ss->subsys_id] = ss;
4909
4910        /*
4911         * no ss->css_alloc seems to need anything important in the ss
4912         * struct, so this can happen first (i.e. before the dummy root
4913         * attachment).
4914         */
4915        css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
4916        if (IS_ERR(css)) {
4917                /* failure case - need to deassign the cgroup_subsys[] slot. */
4918                cgroup_subsys[ss->subsys_id] = NULL;
4919                mutex_unlock(&cgroup_mutex);
4920                return PTR_ERR(css);
4921        }
4922
4923        list_add(&ss->sibling, &cgroup_dummy_root.subsys_list);
4924        ss->root = &cgroup_dummy_root;
4925
4926        /* our new subsystem will be attached to the dummy hierarchy. */
4927        init_css(css, ss, cgroup_dummy_top);
4928        /* init_idr must be after init_css() because it sets css->id. */
4929        if (ss->use_id) {
4930                ret = cgroup_init_idr(ss, css);
4931                if (ret)
4932                        goto err_unload;
4933        }
4934
4935        /*
4936         * Now we need to entangle the css into the existing css_sets. unlike
4937         * in cgroup_init_subsys, there are now multiple css_sets, so each one
4938         * will need a new pointer to it; done by iterating the css_set_table.
4939         * furthermore, modifying the existing css_sets will corrupt the hash
4940         * table state, so each changed css_set will need its hash recomputed.
4941         * this is all done under the css_set_lock.
4942         */
4943        write_lock(&css_set_lock);
4944        hash_for_each_safe(css_set_table, i, tmp, cset, hlist) {
4945                /* skip entries that we already rehashed */
4946                if (cset->subsys[ss->subsys_id])
4947                        continue;
4948                /* remove existing entry */
4949                hash_del(&cset->hlist);
4950                /* set new value */
4951                cset->subsys[ss->subsys_id] = css;
4952                /* recompute hash and restore entry */
4953                key = css_set_hash(cset->subsys);
4954                hash_add(css_set_table, &cset->hlist, key);
4955        }
4956        write_unlock(&css_set_lock);
4957
4958        ret = online_css(css);
4959        if (ret)
4960                goto err_unload;
4961
4962        /* success! */
4963        mutex_unlock(&cgroup_mutex);
4964        return 0;
4965
4966err_unload:
4967        mutex_unlock(&cgroup_mutex);
4968        /* @ss can't be mounted here as try_module_get() would fail */
4969        cgroup_unload_subsys(ss);
4970        return ret;
4971}
4972EXPORT_SYMBOL_GPL(cgroup_load_subsys);
4973
4974/**
4975 * cgroup_unload_subsys: unload a modular subsystem
4976 * @ss: the subsystem to unload
4977 *
4978 * This function should be called in a modular subsystem's exitcall. When this
4979 * function is invoked, the refcount on the subsystem's module will be 0, so
4980 * the subsystem will not be attached to any hierarchy.
4981 */
4982void cgroup_unload_subsys(struct cgroup_subsys *ss)
4983{
4984        struct cgrp_cset_link *link;
4985
4986        BUG_ON(ss->module == NULL);
4987
4988        /*
4989         * we shouldn't be called if the subsystem is in use, and the use of
4990         * try_module_get() in rebind_subsystems() should ensure that it
4991         * doesn't start being used while we're killing it off.
4992         */
4993        BUG_ON(ss->root != &cgroup_dummy_root);
4994
4995        mutex_lock(&cgroup_mutex);
4996
4997        offline_css(cgroup_css(cgroup_dummy_top, ss));
4998
4999        if (ss->use_id)
5000                idr_destroy(&ss->idr);
5001
5002        /* deassign the subsys_id */
5003        cgroup_subsys[ss->subsys_id] = NULL;
5004
5005        /* remove subsystem from the dummy root's list of subsystems */
5006        list_del_init(&ss->sibling);
5007
5008        /*
5009         * disentangle the css from all css_sets attached to the dummy
5010         * top. as in loading, we need to pay our respects to the hashtable
5011         * gods.
5012         */
5013        write_lock(&css_set_lock);
5014        list_for_each_entry(link, &cgroup_dummy_top->cset_links, cset_link) {
5015                struct css_set *cset = link->cset;
5016                unsigned long key;
5017
5018                hash_del(&cset->hlist);
5019                cset->subsys[ss->subsys_id] = NULL;
5020                key = css_set_hash(cset->subsys);
5021                hash_add(css_set_table, &cset->hlist, key);
5022        }
5023        write_unlock(&css_set_lock);
5024
5025        /*
5026         * remove subsystem's css from the cgroup_dummy_top and free it -
5027         * need to free before marking as null because ss->css_free needs
5028         * the cgrp->subsys pointer to find their state. note that this
5029         * also takes care of freeing the css_id.
5030         */
5031        ss->css_free(cgroup_css(cgroup_dummy_top, ss));
5032        RCU_INIT_POINTER(cgroup_dummy_top->subsys[ss->subsys_id], NULL);
5033
5034        mutex_unlock(&cgroup_mutex);
5035}
5036EXPORT_SYMBOL_GPL(cgroup_unload_subsys);
5037
5038/**
5039 * cgroup_init_early - cgroup initialization at system boot
5040 *
5041 * Initialize cgroups at system boot, and initialize any
5042 * subsystems that request early init.
5043 */
5044int __init cgroup_init_early(void)
5045{
5046        struct cgroup_subsys *ss;
5047        int i;
5048
5049        atomic_set(&init_css_set.refcount, 1);
5050        INIT_LIST_HEAD(&init_css_set.cgrp_links);
5051        INIT_LIST_HEAD(&init_css_set.tasks);
5052        INIT_HLIST_NODE(&init_css_set.hlist);
5053        css_set_count = 1;
5054        init_cgroup_root(&cgroup_dummy_root);
5055        cgroup_root_count = 1;
5056        RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
5057
5058        init_cgrp_cset_link.cset = &init_css_set;
5059        init_cgrp_cset_link.cgrp = cgroup_dummy_top;
5060        list_add(&init_cgrp_cset_link.cset_link, &cgroup_dummy_top->cset_links);
5061        list_add(&init_cgrp_cset_link.cgrp_link, &init_css_set.cgrp_links);
5062
5063        /* at bootup time, we don't worry about modular subsystems */
5064        for_each_builtin_subsys(ss, i) {
5065                BUG_ON(!ss->name);
5066                BUG_ON(strlen(ss->name) > MAX_CGROUP_TYPE_NAMELEN);
5067                BUG_ON(!ss->css_alloc);
5068                BUG_ON(!ss->css_free);
5069                if (ss->subsys_id != i) {
5070                        printk(KERN_ERR "cgroup: Subsys %s id == %d\n",
5071                               ss->name, ss->subsys_id);
5072                        BUG();
5073                }
5074
5075                if (ss->early_init)
5076                        cgroup_init_subsys(ss);
5077        }
5078        return 0;
5079}
5080
5081/**
5082 * cgroup_init - cgroup initialization
5083 *
5084 * Register cgroup filesystem and /proc file, and initialize
5085 * any subsystems that didn't request early init.
5086 */
5087int __init cgroup_init(void)
5088{
5089        struct cgroup_subsys *ss;
5090        unsigned long key;
5091        int i, err;
5092
5093        err = bdi_init(&cgroup_backing_dev_info);
5094        if (err)
5095                return err;
5096
5097        for_each_builtin_subsys(ss, i) {
5098                if (!ss->early_init)
5099                        cgroup_init_subsys(ss);
5100                if (ss->use_id)
5101                        cgroup_init_idr(ss, init_css_set.subsys[ss->subsys_id]);
5102        }
5103
5104        /* allocate id for the dummy hierarchy */
5105        mutex_lock(&cgroup_mutex);
5106        mutex_lock(&cgroup_root_mutex);
5107
5108        /* Add init_css_set to the hash table */
5109        key = css_set_hash(init_css_set.subsys);
5110        hash_add(css_set_table, &init_css_set.hlist, key);
5111
5112        BUG_ON(cgroup_init_root_id(&cgroup_dummy_root, 0, 1));
5113
5114        err = idr_alloc(&cgroup_dummy_root.cgroup_idr, cgroup_dummy_top,
5115                        0, 1, GFP_KERNEL);
5116        BUG_ON(err < 0);
5117
5118        mutex_unlock(&cgroup_root_mutex);
5119        mutex_unlock(&cgroup_mutex);
5120
5121        cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
5122        if (!cgroup_kobj) {
5123                err = -ENOMEM;
5124                goto out;
5125        }
5126
5127        err = register_filesystem(&cgroup_fs_type);
5128        if (err < 0) {
5129                kobject_put(cgroup_kobj);
5130                goto out;
5131        }
5132
5133        proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
5134
5135out:
5136        if (err)
5137                bdi_destroy(&cgroup_backing_dev_info);
5138
5139        return err;
5140}
5141
5142/*
5143 * proc_cgroup_show()
5144 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
5145 *  - Used for /proc/<pid>/cgroup.
5146 *  - No need to task_lock(tsk) on this tsk->cgroup reference, as it
5147 *    doesn't really matter if tsk->cgroup changes after we read it,
5148 *    and we take cgroup_mutex, keeping cgroup_attach_task() from changing it
5149 *    anyway.  No need to check that tsk->cgroup != NULL, thanks to
5150 *    the_top_cgroup_hack in cgroup_exit(), which sets an exiting tasks
5151 *    cgroup to top_cgroup.
5152 */
5153
5154/* TODO: Use a proper seq_file iterator */
5155int proc_cgroup_show(struct seq_file *m, void *v)
5156{
5157        struct pid *pid;
5158        struct task_struct *tsk;
5159        char *buf;
5160        int retval;
5161        struct cgroupfs_root *root;
5162
5163        retval = -ENOMEM;
5164        buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
5165        if (!buf)
5166                goto out;
5167
5168        retval = -ESRCH;
5169        pid = m->private;
5170        tsk = get_pid_task(pid, PIDTYPE_PID);
5171        if (!tsk)
5172                goto out_free;
5173
5174        retval = 0;
5175
5176        mutex_lock(&cgroup_mutex);
5177
5178        for_each_active_root(root) {
5179                struct cgroup_subsys *ss;
5180                struct cgroup *cgrp;
5181                int count = 0;
5182
5183                seq_printf(m, "%d:", root->hierarchy_id);
5184                for_each_root_subsys(root, ss)
5185                        seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
5186                if (strlen(root->name))
5187                        seq_printf(m, "%sname=%s", count ? "," : "",
5188                                   root->name);
5189                seq_putc(m, ':');
5190                cgrp = task_cgroup_from_root(tsk, root);
5191                retval = cgroup_path(cgrp, buf, PAGE_SIZE);
5192                if (retval < 0)
5193                        goto out_unlock;
5194                seq_puts(m, buf);
5195                seq_putc(m, '\n');
5196        }
5197
5198out_unlock:
5199        mutex_unlock(&cgroup_mutex);
5200        put_task_struct(tsk);
5201out_free:
5202        kfree(buf);
5203out:
5204        return retval;
5205}
5206
5207/* Display information about each subsystem and each hierarchy */
5208static int proc_cgroupstats_show(struct seq_file *m, void *v)
5209{
5210        struct cgroup_subsys *ss;
5211        int i;
5212
5213        seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
5214        /*
5215         * ideally we don't want subsystems moving around while we do this.
5216         * cgroup_mutex is also necessary to guarantee an atomic snapshot of
5217         * subsys/hierarchy state.
5218         */
5219        mutex_lock(&cgroup_mutex);
5220
5221        for_each_subsys(ss, i)
5222                seq_printf(m, "%s\t%d\t%d\t%d\n",
5223                           ss->name, ss->root->hierarchy_id,
5224                           ss->root->number_of_cgroups, !ss->disabled);
5225
5226        mutex_unlock(&cgroup_mutex);
5227        return 0;
5228}
5229
5230static int cgroupstats_open(struct inode *inode, struct file *file)
5231{
5232        return single_open(file, proc_cgroupstats_show, NULL);
5233}
5234
5235static const struct file_operations proc_cgroupstats_operations = {
5236        .open = cgroupstats_open,
5237        .read = seq_read,
5238        .llseek = seq_lseek,
5239        .release = single_release,
5240};
5241
5242/**
5243 * cgroup_fork - attach newly forked task to its parents cgroup.
5244 * @child: pointer to task_struct of forking parent process.
5245 *
5246 * Description: A task inherits its parent's cgroup at fork().
5247 *
5248 * A pointer to the shared css_set was automatically copied in
5249 * fork.c by dup_task_struct().  However, we ignore that copy, since
5250 * it was not made under the protection of RCU or cgroup_mutex, so
5251 * might no longer be a valid cgroup pointer.  cgroup_attach_task() might
5252 * have already changed current->cgroups, allowing the previously
5253 * referenced cgroup group to be removed and freed.
5254 *
5255 * At the point that cgroup_fork() is called, 'current' is the parent
5256 * task, and the passed argument 'child' points to the child task.
5257 */
5258void cgroup_fork(struct task_struct *child)
5259{
5260        task_lock(current);
5261        get_css_set(task_css_set(current));
5262        child->cgroups = current->cgroups;
5263        task_unlock(current);
5264        INIT_LIST_HEAD(&child->cg_list);
5265}
5266
5267/**
5268 * cgroup_post_fork - called on a new task after adding it to the task list
5269 * @child: the task in question
5270 *
5271 * Adds the task to the list running through its css_set if necessary and
5272 * call the subsystem fork() callbacks.  Has to be after the task is
5273 * visible on the task list in case we race with the first call to
5274 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5275 * list.
5276 */
5277void cgroup_post_fork(struct task_struct *child)
5278{
5279        struct cgroup_subsys *ss;
5280        int i;
5281
5282        /*
5283         * use_task_css_set_links is set to 1 before we walk the tasklist
5284         * under the tasklist_lock and we read it here after we added the child
5285         * to the tasklist under the tasklist_lock as well. If the child wasn't
5286         * yet in the tasklist when we walked through it from
5287         * cgroup_enable_task_cg_lists(), then use_task_css_set_links value
5288         * should be visible now due to the paired locking and barriers implied
5289         * by LOCK/UNLOCK: it is written before the tasklist_lock unlock
5290         * in cgroup_enable_task_cg_lists() and read here after the tasklist_lock
5291         * lock on fork.
5292         */
5293        if (use_task_css_set_links) {
5294                write_lock(&css_set_lock);
5295                task_lock(child);
5296                if (list_empty(&child->cg_list))
5297                        list_add(&child->cg_list, &task_css_set(child)->tasks);
5298                task_unlock(child);
5299                write_unlock(&css_set_lock);
5300        }
5301
5302        /*
5303         * Call ss->fork().  This must happen after @child is linked on
5304         * css_set; otherwise, @child might change state between ->fork()
5305         * and addition to css_set.
5306         */
5307        if (need_forkexit_callback) {
5308                /*
5309                 * fork/exit callbacks are supported only for builtin
5310                 * subsystems, and the builtin section of the subsys
5311                 * array is immutable, so we don't need to lock the
5312                 * subsys array here. On the other hand, modular section
5313                 * of the array can be freed at module unload, so we
5314                 * can't touch that.
5315                 */
5316                for_each_builtin_subsys(ss, i)
5317                        if (ss->fork)
5318                                ss->fork(child);
5319        }
5320}
5321
5322/**
5323 * cgroup_exit - detach cgroup from exiting task
5324 * @tsk: pointer to task_struct of exiting process
5325 * @run_callback: run exit callbacks?
5326 *
5327 * Description: Detach cgroup from @tsk and release it.
5328 *
5329 * Note that cgroups marked notify_on_release force every task in
5330 * them to take the global cgroup_mutex mutex when exiting.
5331 * This could impact scaling on very large systems.  Be reluctant to
5332 * use notify_on_release cgroups where very high task exit scaling
5333 * is required on large systems.
5334 *
5335 * the_top_cgroup_hack:
5336 *
5337 *    Set the exiting tasks cgroup to the root cgroup (top_cgroup).
5338 *
5339 *    We call cgroup_exit() while the task is still competent to
5340 *    handle notify_on_release(), then leave the task attached to the
5341 *    root cgroup in each hierarchy for the remainder of its exit.
5342 *
5343 *    To do this properly, we would increment the reference count on
5344 *    top_cgroup, and near the very end of the kernel/exit.c do_exit()
5345 *    code we would add a second cgroup function call, to drop that
5346 *    reference.  This would just create an unnecessary hot spot on
5347 *    the top_cgroup reference count, to no avail.
5348 *
5349 *    Normally, holding a reference to a cgroup without bumping its
5350 *    count is unsafe.   The cgroup could go away, or someone could
5351 *    attach us to a different cgroup, decrementing the count on
5352 *    the first cgroup that we never incremented.  But in this case,
5353 *    top_cgroup isn't going away, and either task has PF_EXITING set,
5354 *    which wards off any cgroup_attach_task() attempts, or task is a failed
5355 *    fork, never visible to cgroup_attach_task.
5356 */
5357void cgroup_exit(struct task_struct *tsk, int run_callbacks)
5358{
5359        struct cgroup_subsys *ss;
5360        struct css_set *cset;
5361        int i;
5362
5363        /*
5364         * Unlink from the css_set task list if necessary.
5365         * Optimistically check cg_list before taking
5366         * css_set_lock
5367         */
5368        if (!list_empty(&tsk->cg_list)) {
5369                write_lock(&css_set_lock);
5370                if (!list_empty(&tsk->cg_list))
5371                        list_del_init(&tsk->cg_list);
5372                write_unlock(&css_set_lock);
5373        }
5374
5375        /* Reassign the task to the init_css_set. */
5376        task_lock(tsk);
5377        cset = task_css_set(tsk);
5378        RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
5379
5380        if (run_callbacks && need_forkexit_callback) {
5381                /*
5382                 * fork/exit callbacks are supported only for builtin
5383                 * subsystems, see cgroup_post_fork() for details.
5384                 */
5385                for_each_builtin_subsys(ss, i) {
5386                        if (ss->exit) {
5387                                struct cgroup_subsys_state *old_css = cset->subsys[i];
5388                                struct cgroup_subsys_state *css = task_css(tsk, i);
5389
5390                                ss->exit(css, old_css, tsk);
5391                        }
5392                }
5393        }
5394        task_unlock(tsk);
5395
5396        put_css_set_taskexit(cset);
5397}
5398
5399static void check_for_release(struct cgroup *cgrp)
5400{
5401        if (cgroup_is_releasable(cgrp) &&
5402            list_empty(&cgrp->cset_links) && list_empty(&cgrp->children)) {
5403                /*
5404                 * Control Group is currently removeable. If it's not
5405                 * already queued for a userspace notification, queue
5406                 * it now
5407                 */
5408                int need_schedule_work = 0;
5409
5410                raw_spin_lock(&release_list_lock);
5411                if (!cgroup_is_dead(cgrp) &&
5412                    list_empty(&cgrp->release_list)) {
5413                        list_add(&cgrp->release_list, &release_list);
5414                        need_schedule_work = 1;
5415                }
5416                raw_spin_unlock(&release_list_lock);
5417                if (need_schedule_work)
5418                        schedule_work(&release_agent_work);
5419        }
5420}
5421
5422/*
5423 * Notify userspace when a cgroup is released, by running the
5424 * configured release agent with the name of the cgroup (path
5425 * relative to the root of cgroup file system) as the argument.
5426 *
5427 * Most likely, this user command will try to rmdir this cgroup.
5428 *
5429 * This races with the possibility that some other task will be
5430 * attached to this cgroup before it is removed, or that some other
5431 * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
5432 * The presumed 'rmdir' will fail quietly if this cgroup is no longer
5433 * unused, and this cgroup will be reprieved from its death sentence,
5434 * to continue to serve a useful existence.  Next time it's released,
5435 * we will get notified again, if it still has 'notify_on_release' set.
5436 *
5437 * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
5438 * means only wait until the task is successfully execve()'d.  The
5439 * separate release agent task is forked by call_usermodehelper(),
5440 * then control in this thread returns here, without waiting for the
5441 * release agent task.  We don't bother to wait because the caller of
5442 * this routine has no use for the exit status of the release agent
5443 * task, so no sense holding our caller up for that.
5444 */
5445static void cgroup_release_agent(struct work_struct *work)
5446{
5447        BUG_ON(work != &release_agent_work);
5448        mutex_lock(&cgroup_mutex);
5449        raw_spin_lock(&release_list_lock);
5450        while (!list_empty(&release_list)) {
5451                char *argv[3], *envp[3];
5452                int i;
5453                char *pathbuf = NULL, *agentbuf = NULL;
5454                struct cgroup *cgrp = list_entry(release_list.next,
5455                                                    struct cgroup,
5456                                                    release_list);
5457                list_del_init(&cgrp->release_list);
5458                raw_spin_unlock(&release_list_lock);
5459                pathbuf = kmalloc(PAGE_SIZE, GFP_KERNEL);
5460                if (!pathbuf)
5461                        goto continue_free;
5462                if (cgroup_path(cgrp, pathbuf, PAGE_SIZE) < 0)
5463                        goto continue_free;
5464                agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
5465                if (!agentbuf)
5466                        goto continue_free;
5467
5468                i = 0;
5469                argv[i++] = agentbuf;
5470                argv[i++] = pathbuf;
5471                argv[i] = NULL;
5472
5473                i = 0;
5474                /* minimal command environment */
5475                envp[i++] = "HOME=/";
5476                envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
5477                envp[i] = NULL;
5478
5479                /* Drop the lock while we invoke the usermode helper,
5480                 * since the exec could involve hitting disk and hence
5481                 * be a slow process */
5482                mutex_unlock(&cgroup_mutex);
5483                call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
5484                mutex_lock(&cgroup_mutex);
5485 continue_free:
5486                kfree(pathbuf);
5487                kfree(agentbuf);
5488                raw_spin_lock(&release_list_lock);
5489        }
5490        raw_spin_unlock(&release_list_lock);
5491        mutex_unlock(&cgroup_mutex);
5492}
5493
5494static int __init cgroup_disable(char *str)
5495{
5496        struct cgroup_subsys *ss;
5497        char *token;
5498        int i;
5499
5500        while ((token = strsep(&str, ",")) != NULL) {
5501                if (!*token)
5502                        continue;
5503
5504                /*
5505                 * cgroup_disable, being at boot time, can't know about
5506                 * module subsystems, so we don't worry about them.
5507                 */
5508                for_each_builtin_subsys(ss, i) {
5509                        if (!strcmp(token, ss->name)) {
5510                                ss->disabled = 1;
5511                                printk(KERN_INFO "Disabling %s control group"
5512                                        " subsystem\n", ss->name);
5513                                break;
5514                        }
5515                }
5516        }
5517        return 1;
5518}
5519__setup("cgroup_disable=", cgroup_disable);
5520
5521/*
5522 * Functons for CSS ID.
5523 */
5524
5525/* to get ID other than 0, this should be called when !cgroup_is_dead() */
5526unsigned short css_id(struct cgroup_subsys_state *css)
5527{
5528        struct css_id *cssid;
5529
5530        /*
5531         * This css_id() can return correct value when somone has refcnt
5532         * on this or this is under rcu_read_lock(). Once css->id is allocated,
5533         * it's unchanged until freed.
5534         */
5535        cssid = rcu_dereference_raw(css->id);
5536
5537        if (cssid)
5538                return cssid->id;
5539        return 0;
5540}
5541EXPORT_SYMBOL_GPL(css_id);
5542
5543/**
5544 *  css_is_ancestor - test "root" css is an ancestor of "child"
5545 * @child: the css to be tested.
5546 * @root: the css supporsed to be an ancestor of the child.
5547 *
5548 * Returns true if "root" is an ancestor of "child" in its hierarchy. Because
5549 * this function reads css->id, the caller must hold rcu_read_lock().
5550 * But, considering usual usage, the csses should be valid objects after test.
5551 * Assuming that the caller will do some action to the child if this returns
5552 * returns true, the caller must take "child";s reference count.
5553 * If "child" is valid object and this returns true, "root" is valid, too.
5554 */
5555
5556bool css_is_ancestor(struct cgroup_subsys_state *child,
5557                    const struct cgroup_subsys_state *root)
5558{
5559        struct css_id *child_id;
5560        struct css_id *root_id;
5561
5562        child_id  = rcu_dereference(child->id);
5563        if (!child_id)
5564                return false;
5565        root_id = rcu_dereference(root->id);
5566        if (!root_id)
5567                return false;
5568        if (child_id->depth < root_id->depth)
5569                return false;
5570        if (child_id->stack[root_id->depth] != root_id->id)
5571                return false;
5572        return true;
5573}
5574
5575void free_css_id(struct cgroup_subsys *ss, struct cgroup_subsys_state *css)
5576{
5577        struct css_id *id = rcu_dereference_protected(css->id, true);
5578
5579        /* When this is called before css_id initialization, id can be NULL */
5580        if (!id)
5581                return;
5582
5583        BUG_ON(!ss->use_id);
5584
5585        rcu_assign_pointer(id->css, NULL);
5586        rcu_assign_pointer(css->id, NULL);
5587        spin_lock(&ss->id_lock);
5588        idr_remove(&ss->idr, id->id);
5589        spin_unlock(&ss->id_lock);
5590        kfree_rcu(id, rcu_head);
5591}
5592EXPORT_SYMBOL_GPL(free_css_id);
5593
5594/*
5595 * This is called by init or create(). Then, calls to this function are
5596 * always serialized (By cgroup_mutex() at create()).
5597 */
5598
5599static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth)
5600{
5601        struct css_id *newid;
5602        int ret, size;
5603
5604        BUG_ON(!ss->use_id);
5605
5606        size = sizeof(*newid) + sizeof(unsigned short) * (depth + 1);
5607        newid = kzalloc(size, GFP_KERNEL);
5608        if (!newid)
5609                return ERR_PTR(-ENOMEM);
5610
5611        idr_preload(GFP_KERNEL);
5612        spin_lock(&ss->id_lock);
5613        /* Don't use 0. allocates an ID of 1-65535 */
5614        ret = idr_alloc(&ss->idr, newid, 1, CSS_ID_MAX + 1, GFP_NOWAIT);
5615        spin_unlock(&ss->id_lock);
5616        idr_preload_end();
5617
5618        /* Returns error when there are no free spaces for new ID.*/
5619        if (ret < 0)
5620                goto err_out;
5621
5622        newid->id = ret;
5623        newid->depth = depth;
5624        return newid;
5625err_out:
5626        kfree(newid);
5627        return ERR_PTR(ret);
5628
5629}
5630
5631static int __init_or_module cgroup_init_idr(struct cgroup_subsys *ss,
5632                                            struct cgroup_subsys_state *rootcss)
5633{
5634        struct css_id *newid;
5635
5636        spin_lock_init(&ss->id_lock);
5637        idr_init(&ss->idr);
5638
5639        newid = get_new_cssid(ss, 0);
5640        if (IS_ERR(newid))
5641                return PTR_ERR(newid);
5642
5643        newid->stack[0] = newid->id;
5644        RCU_INIT_POINTER(newid->css, rootcss);
5645        RCU_INIT_POINTER(rootcss->id, newid);
5646        return 0;
5647}
5648
5649static int alloc_css_id(struct cgroup_subsys_state *child_css)
5650{
5651        struct cgroup_subsys_state *parent_css = css_parent(child_css);
5652        struct css_id *child_id, *parent_id;
5653        int i, depth;
5654
5655        parent_id = rcu_dereference_protected(parent_css->id, true);
5656        depth = parent_id->depth + 1;
5657
5658        child_id = get_new_cssid(child_css->ss, depth);
5659        if (IS_ERR(child_id))
5660                return PTR_ERR(child_id);
5661
5662        for (i = 0; i < depth; i++)
5663                child_id->stack[i] = parent_id->stack[i];
5664        child_id->stack[depth] = child_id->id;
5665        /*
5666