linux/kernel/async.c
<<
>>
Prefs
   1/*
   2 * async.c: Asynchronous function calls for boot performance
   3 *
   4 * (C) Copyright 2009 Intel Corporation
   5 * Author: Arjan van de Ven <arjan@linux.intel.com>
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License
   9 * as published by the Free Software Foundation; version 2
  10 * of the License.
  11 */
  12
  13
  14/*
  15
  16Goals and Theory of Operation
  17
  18The primary goal of this feature is to reduce the kernel boot time,
  19by doing various independent hardware delays and discovery operations
  20decoupled and not strictly serialized.
  21
  22More specifically, the asynchronous function call concept allows
  23certain operations (primarily during system boot) to happen
  24asynchronously, out of order, while these operations still
  25have their externally visible parts happen sequentially and in-order.
  26(not unlike how out-of-order CPUs retire their instructions in order)
  27
  28Key to the asynchronous function call implementation is the concept of
  29a "sequence cookie" (which, although it has an abstracted type, can be
  30thought of as a monotonically incrementing number).
  31
  32The async core will assign each scheduled event such a sequence cookie and
  33pass this to the called functions.
  34
  35The asynchronously called function should before doing a globally visible
  36operation, such as registering device numbers, call the
  37async_synchronize_cookie() function and pass in its own cookie. The
  38async_synchronize_cookie() function will make sure that all asynchronous
  39operations that were scheduled prior to the operation corresponding with the
  40cookie have completed.
  41
  42Subsystem/driver initialization code that scheduled asynchronous probe
  43functions, but which shares global resources with other drivers/subsystems
  44that do not use the asynchronous call feature, need to do a full
  45synchronization with the async_synchronize_full() function, before returning
  46from their init function. This is to maintain strict ordering between the
  47asynchronous and synchronous parts of the kernel.
  48
  49*/
  50
  51#include <linux/async.h>
  52#include <linux/atomic.h>
  53#include <linux/ktime.h>
  54#include <linux/export.h>
  55#include <linux/wait.h>
  56#include <linux/sched.h>
  57#include <linux/slab.h>
  58#include <linux/workqueue.h>
  59
  60#include "workqueue_internal.h"
  61
  62static async_cookie_t next_cookie = 1;
  63
  64#define MAX_WORK                32768
  65#define ASYNC_COOKIE_MAX        ULLONG_MAX      /* infinity cookie */
  66
  67static LIST_HEAD(async_global_pending); /* pending from all registered doms */
  68static ASYNC_DOMAIN(async_dfl_domain);
  69static DEFINE_SPINLOCK(async_lock);
  70
  71struct async_entry {
  72        struct list_head        domain_list;
  73        struct list_head        global_list;
  74        struct work_struct      work;
  75        async_cookie_t          cookie;
  76        async_func_ptr          *func;
  77        void                    *data;
  78        struct async_domain     *domain;
  79};
  80
  81static DECLARE_WAIT_QUEUE_HEAD(async_done);
  82
  83static atomic_t entry_count;
  84
  85static async_cookie_t lowest_in_progress(struct async_domain *domain)
  86{
  87        struct async_entry *first = NULL;
  88        async_cookie_t ret = ASYNC_COOKIE_MAX;
  89        unsigned long flags;
  90
  91        spin_lock_irqsave(&async_lock, flags);
  92
  93        if (domain) {
  94                if (!list_empty(&domain->pending))
  95                        first = list_first_entry(&domain->pending,
  96                                        struct async_entry, domain_list);
  97        } else {
  98                if (!list_empty(&async_global_pending))
  99                        first = list_first_entry(&async_global_pending,
 100                                        struct async_entry, global_list);
 101        }
 102
 103        if (first)
 104                ret = first->cookie;
 105
 106        spin_unlock_irqrestore(&async_lock, flags);
 107        return ret;
 108}
 109
 110/*
 111 * pick the first pending entry and run it
 112 */
 113static void async_run_entry_fn(struct work_struct *work)
 114{
 115        struct async_entry *entry =
 116                container_of(work, struct async_entry, work);
 117        unsigned long flags;
 118        ktime_t uninitialized_var(calltime), delta, rettime;
 119
 120        /* 1) run (and print duration) */
 121        if (initcall_debug && system_state == SYSTEM_BOOTING) {
 122                printk(KERN_DEBUG "calling  %lli_%pF @ %i\n",
 123                        (long long)entry->cookie,
 124                        entry->func, task_pid_nr(current));
 125                calltime = ktime_get();
 126        }
 127        entry->func(entry->data, entry->cookie);
 128        if (initcall_debug && system_state == SYSTEM_BOOTING) {
 129                rettime = ktime_get();
 130                delta = ktime_sub(rettime, calltime);
 131                printk(KERN_DEBUG "initcall %lli_%pF returned 0 after %lld usecs\n",
 132                        (long long)entry->cookie,
 133                        entry->func,
 134                        (long long)ktime_to_ns(delta) >> 10);
 135        }
 136
 137        /* 2) remove self from the pending queues */
 138        spin_lock_irqsave(&async_lock, flags);
 139        list_del_init(&entry->domain_list);
 140        list_del_init(&entry->global_list);
 141
 142        /* 3) free the entry */
 143        kfree(entry);
 144        atomic_dec(&entry_count);
 145
 146        spin_unlock_irqrestore(&async_lock, flags);
 147
 148        /* 4) wake up any waiters */
 149        wake_up(&async_done);
 150}
 151
 152static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct async_domain *domain)
 153{
 154        struct async_entry *entry;
 155        unsigned long flags;
 156        async_cookie_t newcookie;
 157
 158        /* allow irq-off callers */
 159        entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
 160
 161        /*
 162         * If we're out of memory or if there's too much work
 163         * pending already, we execute synchronously.
 164         */
 165        if (!entry || atomic_read(&entry_count) > MAX_WORK) {
 166                kfree(entry);
 167                spin_lock_irqsave(&async_lock, flags);
 168                newcookie = next_cookie++;
 169                spin_unlock_irqrestore(&async_lock, flags);
 170
 171                /* low on memory.. run synchronously */
 172                ptr(data, newcookie);
 173                return newcookie;
 174        }
 175        INIT_LIST_HEAD(&entry->domain_list);
 176        INIT_LIST_HEAD(&entry->global_list);
 177        INIT_WORK(&entry->work, async_run_entry_fn);
 178        entry->func = ptr;
 179        entry->data = data;
 180        entry->domain = domain;
 181
 182        spin_lock_irqsave(&async_lock, flags);
 183
 184        /* allocate cookie and queue */
 185        newcookie = entry->cookie = next_cookie++;
 186
 187        list_add_tail(&entry->domain_list, &domain->pending);
 188        if (domain->registered)
 189                list_add_tail(&entry->global_list, &async_global_pending);
 190
 191        atomic_inc(&entry_count);
 192        spin_unlock_irqrestore(&async_lock, flags);
 193
 194        /* mark that this task has queued an async job, used by module init */
 195        current->flags |= PF_USED_ASYNC;
 196
 197        /* schedule for execution */
 198        queue_work(system_unbound_wq, &entry->work);
 199
 200        return newcookie;
 201}
 202
 203/**
 204 * async_schedule - schedule a function for asynchronous execution
 205 * @ptr: function to execute asynchronously
 206 * @data: data pointer to pass to the function
 207 *
 208 * Returns an async_cookie_t that may be used for checkpointing later.
 209 * Note: This function may be called from atomic or non-atomic contexts.
 210 */
 211async_cookie_t async_schedule(async_func_ptr *ptr, void *data)
 212{
 213        return __async_schedule(ptr, data, &async_dfl_domain);
 214}
 215EXPORT_SYMBOL_GPL(async_schedule);
 216
 217/**
 218 * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
 219 * @ptr: function to execute asynchronously
 220 * @data: data pointer to pass to the function
 221 * @domain: the domain
 222 *
 223 * Returns an async_cookie_t that may be used for checkpointing later.
 224 * @domain may be used in the async_synchronize_*_domain() functions to
 225 * wait within a certain synchronization domain rather than globally.  A
 226 * synchronization domain is specified via @domain.  Note: This function
 227 * may be called from atomic or non-atomic contexts.
 228 */
 229async_cookie_t async_schedule_domain(async_func_ptr *ptr, void *data,
 230                                     struct async_domain *domain)
 231{
 232        return __async_schedule(ptr, data, domain);
 233}
 234EXPORT_SYMBOL_GPL(async_schedule_domain);
 235
 236/**
 237 * async_synchronize_full - synchronize all asynchronous function calls
 238 *
 239 * This function waits until all asynchronous function calls have been done.
 240 */
 241void async_synchronize_full(void)
 242{
 243        async_synchronize_full_domain(NULL);
 244}
 245EXPORT_SYMBOL_GPL(async_synchronize_full);
 246
 247/**
 248 * async_unregister_domain - ensure no more anonymous waiters on this domain
 249 * @domain: idle domain to flush out of any async_synchronize_full instances
 250 *
 251 * async_synchronize_{cookie|full}_domain() are not flushed since callers
 252 * of these routines should know the lifetime of @domain
 253 *
 254 * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
 255 */
 256void async_unregister_domain(struct async_domain *domain)
 257{
 258        spin_lock_irq(&async_lock);
 259        WARN_ON(!domain->registered || !list_empty(&domain->pending));
 260        domain->registered = 0;
 261        spin_unlock_irq(&async_lock);
 262}
 263EXPORT_SYMBOL_GPL(async_unregister_domain);
 264
 265/**
 266 * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
 267 * @domain: the domain to synchronize
 268 *
 269 * This function waits until all asynchronous function calls for the
 270 * synchronization domain specified by @domain have been done.
 271 */
 272void async_synchronize_full_domain(struct async_domain *domain)
 273{
 274        async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
 275}
 276EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
 277
 278/**
 279 * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
 280 * @cookie: async_cookie_t to use as checkpoint
 281 * @domain: the domain to synchronize (%NULL for all registered domains)
 282 *
 283 * This function waits until all asynchronous function calls for the
 284 * synchronization domain specified by @domain submitted prior to @cookie
 285 * have been done.
 286 */
 287void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
 288{
 289        ktime_t uninitialized_var(starttime), delta, endtime;
 290
 291        if (initcall_debug && system_state == SYSTEM_BOOTING) {
 292                printk(KERN_DEBUG "async_waiting @ %i\n", task_pid_nr(current));
 293                starttime = ktime_get();
 294        }
 295
 296        wait_event(async_done, lowest_in_progress(domain) >= cookie);
 297
 298        if (initcall_debug && system_state == SYSTEM_BOOTING) {
 299                endtime = ktime_get();
 300                delta = ktime_sub(endtime, starttime);
 301
 302                printk(KERN_DEBUG "async_continuing @ %i after %lli usec\n",
 303                        task_pid_nr(current),
 304                        (long long)ktime_to_ns(delta) >> 10);
 305        }
 306}
 307EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
 308
 309/**
 310 * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
 311 * @cookie: async_cookie_t to use as checkpoint
 312 *
 313 * This function waits until all asynchronous function calls prior to @cookie
 314 * have been done.
 315 */
 316void async_synchronize_cookie(async_cookie_t cookie)
 317{
 318        async_synchronize_cookie_domain(cookie, &async_dfl_domain);
 319}
 320EXPORT_SYMBOL_GPL(async_synchronize_cookie);
 321
 322/**
 323 * current_is_async - is %current an async worker task?
 324 *
 325 * Returns %true if %current is an async worker task.
 326 */
 327bool current_is_async(void)
 328{
 329        struct worker *worker = current_wq_worker();
 330
 331        return worker && worker->current_func == async_run_entry_fn;
 332}
 333
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.