linux/drivers/cpufreq/cpufreq.c
<<
>>
Prefs
   1/*
   2 *  linux/drivers/cpufreq/cpufreq.c
   3 *
   4 *  Copyright (C) 2001 Russell King
   5 *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6 *
   7 *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8 *      Added handling for CPU hotplug
   9 *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10 *      Fix handling for CPU hotplug -- affected CPUs
  11 *
  12 * This program is free software; you can redistribute it and/or modify
  13 * it under the terms of the GNU General Public License version 2 as
  14 * published by the Free Software Foundation.
  15 *
  16 */
  17
  18#include <linux/kernel.h>
  19#include <linux/module.h>
  20#include <linux/init.h>
  21#include <linux/notifier.h>
  22#include <linux/cpufreq.h>
  23#include <linux/delay.h>
  24#include <linux/interrupt.h>
  25#include <linux/spinlock.h>
  26#include <linux/device.h>
  27#include <linux/slab.h>
  28#include <linux/cpu.h>
  29#include <linux/completion.h>
  30#include <linux/mutex.h>
  31
  32#define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                "cpufreq-core", msg)
  34
  35/**
  36 * The "cpufreq driver" - the arch- or hardware-dependent low
  37 * level driver of CPUFreq support, and its spinlock. This lock
  38 * also protects the cpufreq_cpu_data array.
  39 */
  40static struct cpufreq_driver *cpufreq_driver;
  41static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42#ifdef CONFIG_HOTPLUG_CPU
  43/* This one keeps track of the previously set governor of a removed CPU */
  44static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45#endif
  46static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48/*
  49 * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50 * all cpufreq/hotplug/workqueue/etc related lock issues.
  51 *
  52 * The rules for this semaphore:
  53 * - Any routine that wants to read from the policy structure will
  54 *   do a down_read on this semaphore.
  55 * - Any routine that will write to the policy structure and/or may take away
  56 *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57 *   mode before doing so.
  58 *
  59 * Additional rules:
  60 * - All holders of the lock should check to make sure that the CPU they
  61 *   are concerned with are online after they get the lock.
  62 * - Governor routines that can be called in cpufreq hotplug path should not
  63 *   take this sem as top level hotplug notifier handler takes this.
  64 */
  65static DEFINE_PER_CPU(int, policy_cpu);
  66static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68#define lock_policy_rwsem(mode, cpu)                                    \
  69int lock_policy_rwsem_##mode                                            \
  70(int cpu)                                                               \
  71{                                                                       \
  72        int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73        BUG_ON(policy_cpu == -1);                                       \
  74        down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75        if (unlikely(!cpu_online(cpu))) {                               \
  76                up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                return -1;                                              \
  78        }                                                               \
  79                                                                        \
  80        return 0;                                                       \
  81}
  82
  83lock_policy_rwsem(read, cpu);
  84EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86lock_policy_rwsem(write, cpu);
  87EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89void unlock_policy_rwsem_read(int cpu)
  90{
  91        int policy_cpu = per_cpu(policy_cpu, cpu);
  92        BUG_ON(policy_cpu == -1);
  93        up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94}
  95EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97void unlock_policy_rwsem_write(int cpu)
  98{
  99        int policy_cpu = per_cpu(policy_cpu, cpu);
 100        BUG_ON(policy_cpu == -1);
 101        up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102}
 103EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106/* internal prototypes */
 107static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108static unsigned int __cpufreq_get(unsigned int cpu);
 109static void handle_update(struct work_struct *work);
 110
 111/**
 112 * Two notifier lists: the "policy" list is involved in the
 113 * validation process for a new CPU frequency policy; the
 114 * "transition" list for kernel code that needs to handle
 115 * changes to devices when the CPU clock speed changes.
 116 * The mutex locks both lists.
 117 */
 118static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121static bool init_cpufreq_transition_notifier_list_called;
 122static int __init init_cpufreq_transition_notifier_list(void)
 123{
 124        srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 125        init_cpufreq_transition_notifier_list_called = true;
 126        return 0;
 127}
 128pure_initcall(init_cpufreq_transition_notifier_list);
 129
 130static LIST_HEAD(cpufreq_governor_list);
 131static DEFINE_MUTEX (cpufreq_governor_mutex);
 132
 133struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 134{
 135        struct cpufreq_policy *data;
 136        unsigned long flags;
 137
 138        if (cpu >= nr_cpu_ids)
 139                goto err_out;
 140
 141        /* get the cpufreq driver */
 142        spin_lock_irqsave(&cpufreq_driver_lock, flags);
 143
 144        if (!cpufreq_driver)
 145                goto err_out_unlock;
 146
 147        if (!try_module_get(cpufreq_driver->owner))
 148                goto err_out_unlock;
 149
 150
 151        /* get the CPU */
 152        data = per_cpu(cpufreq_cpu_data, cpu);
 153
 154        if (!data)
 155                goto err_out_put_module;
 156
 157        if (!kobject_get(&data->kobj))
 158                goto err_out_put_module;
 159
 160        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 161        return data;
 162
 163err_out_put_module:
 164        module_put(cpufreq_driver->owner);
 165err_out_unlock:
 166        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 167err_out:
 168        return NULL;
 169}
 170EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 171
 172
 173void cpufreq_cpu_put(struct cpufreq_policy *data)
 174{
 175        kobject_put(&data->kobj);
 176        module_put(cpufreq_driver->owner);
 177}
 178EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 179
 180
 181/*********************************************************************
 182 *                     UNIFIED DEBUG HELPERS                         *
 183 *********************************************************************/
 184#ifdef CONFIG_CPU_FREQ_DEBUG
 185
 186/* what part(s) of the CPUfreq subsystem are debugged? */
 187static unsigned int debug;
 188
 189/* is the debug output ratelimit'ed using printk_ratelimit? User can
 190 * set or modify this value.
 191 */
 192static unsigned int debug_ratelimit = 1;
 193
 194/* is the printk_ratelimit'ing enabled? It's enabled after a successful
 195 * loading of a cpufreq driver, temporarily disabled when a new policy
 196 * is set, and disabled upon cpufreq driver removal
 197 */
 198static unsigned int disable_ratelimit = 1;
 199static DEFINE_SPINLOCK(disable_ratelimit_lock);
 200
 201static void cpufreq_debug_enable_ratelimit(void)
 202{
 203        unsigned long flags;
 204
 205        spin_lock_irqsave(&disable_ratelimit_lock, flags);
 206        if (disable_ratelimit)
 207                disable_ratelimit--;
 208        spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 209}
 210
 211static void cpufreq_debug_disable_ratelimit(void)
 212{
 213        unsigned long flags;
 214
 215        spin_lock_irqsave(&disable_ratelimit_lock, flags);
 216        disable_ratelimit++;
 217        spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 218}
 219
 220void cpufreq_debug_printk(unsigned int type, const char *prefix,
 221                        const char *fmt, ...)
 222{
 223        char s[256];
 224        va_list args;
 225        unsigned int len;
 226        unsigned long flags;
 227
 228        WARN_ON(!prefix);
 229        if (type & debug) {
 230                spin_lock_irqsave(&disable_ratelimit_lock, flags);
 231                if (!disable_ratelimit && debug_ratelimit
 232                                        && !printk_ratelimit()) {
 233                        spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 234                        return;
 235                }
 236                spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 237
 238                len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 239
 240                va_start(args, fmt);
 241                len += vsnprintf(&s[len], (256 - len), fmt, args);
 242                va_end(args);
 243
 244                printk(s);
 245
 246                WARN_ON(len < 5);
 247        }
 248}
 249EXPORT_SYMBOL(cpufreq_debug_printk);
 250
 251
 252module_param(debug, uint, 0644);
 253MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 254                        " 2 to debug drivers, and 4 to debug governors.");
 255
 256module_param(debug_ratelimit, uint, 0644);
 257MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 258                                        " set to 0 to disable ratelimiting.");
 259
 260#else /* !CONFIG_CPU_FREQ_DEBUG */
 261
 262static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 263static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 264
 265#endif /* CONFIG_CPU_FREQ_DEBUG */
 266
 267
 268/*********************************************************************
 269 *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 270 *********************************************************************/
 271
 272/**
 273 * adjust_jiffies - adjust the system "loops_per_jiffy"
 274 *
 275 * This function alters the system "loops_per_jiffy" for the clock
 276 * speed change. Note that loops_per_jiffy cannot be updated on SMP
 277 * systems as each CPU might be scaled differently. So, use the arch
 278 * per-CPU loops_per_jiffy value wherever possible.
 279 */
 280#ifndef CONFIG_SMP
 281static unsigned long l_p_j_ref;
 282static unsigned int  l_p_j_ref_freq;
 283
 284static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 285{
 286        if (ci->flags & CPUFREQ_CONST_LOOPS)
 287                return;
 288
 289        if (!l_p_j_ref_freq) {
 290                l_p_j_ref = loops_per_jiffy;
 291                l_p_j_ref_freq = ci->old;
 292                dprintk("saving %lu as reference value for loops_per_jiffy; "
 293                        "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 294        }
 295        if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 296            (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 297            (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 298                loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 299                                                                ci->new);
 300                dprintk("scaling loops_per_jiffy to %lu "
 301                        "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 302        }
 303}
 304#else
 305static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 306{
 307        return;
 308}
 309#endif
 310
 311
 312/**
 313 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 314 * on frequency transition.
 315 *
 316 * This function calls the transition notifiers and the "adjust_jiffies"
 317 * function. It is called twice on all CPU frequency changes that have
 318 * external effects.
 319 */
 320void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 321{
 322        struct cpufreq_policy *policy;
 323
 324        BUG_ON(irqs_disabled());
 325
 326        freqs->flags = cpufreq_driver->flags;
 327        dprintk("notification %u of frequency transition to %u kHz\n",
 328                state, freqs->new);
 329
 330        policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 331        switch (state) {
 332
 333        case CPUFREQ_PRECHANGE:
 334                /* detect if the driver reported a value as "old frequency"
 335                 * which is not equal to what the cpufreq core thinks is
 336                 * "old frequency".
 337                 */
 338                if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 339                        if ((policy) && (policy->cpu == freqs->cpu) &&
 340                            (policy->cur) && (policy->cur != freqs->old)) {
 341                                dprintk("Warning: CPU frequency is"
 342                                        " %u, cpufreq assumed %u kHz.\n",
 343                                        freqs->old, policy->cur);
 344                                freqs->old = policy->cur;
 345                        }
 346                }
 347                srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 348                                CPUFREQ_PRECHANGE, freqs);
 349                adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 350                break;
 351
 352        case CPUFREQ_POSTCHANGE:
 353                adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 354                srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 355                                CPUFREQ_POSTCHANGE, freqs);
 356                if (likely(policy) && likely(policy->cpu == freqs->cpu))
 357                        policy->cur = freqs->new;
 358                break;
 359        }
 360}
 361EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 362
 363
 364
 365/*********************************************************************
 366 *                          SYSFS INTERFACE                          *
 367 *********************************************************************/
 368
 369static struct cpufreq_governor *__find_governor(const char *str_governor)
 370{
 371        struct cpufreq_governor *t;
 372
 373        list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 374                if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 375                        return t;
 376
 377        return NULL;
 378}
 379
 380/**
 381 * cpufreq_parse_governor - parse a governor string
 382 */
 383static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 384                                struct cpufreq_governor **governor)
 385{
 386        int err = -EINVAL;
 387
 388        if (!cpufreq_driver)
 389                goto out;
 390
 391        if (cpufreq_driver->setpolicy) {
 392                if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 393                        *policy = CPUFREQ_POLICY_PERFORMANCE;
 394                        err = 0;
 395                } else if (!strnicmp(str_governor, "powersave",
 396                                                CPUFREQ_NAME_LEN)) {
 397                        *policy = CPUFREQ_POLICY_POWERSAVE;
 398                        err = 0;
 399                }
 400        } else if (cpufreq_driver->target) {
 401                struct cpufreq_governor *t;
 402
 403                mutex_lock(&cpufreq_governor_mutex);
 404
 405                t = __find_governor(str_governor);
 406
 407                if (t == NULL) {
 408                        char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 409                                                                str_governor);
 410
 411                        if (name) {
 412                                int ret;
 413
 414                                mutex_unlock(&cpufreq_governor_mutex);
 415                                ret = request_module("%s", name);
 416                                mutex_lock(&cpufreq_governor_mutex);
 417
 418                                if (ret == 0)
 419                                        t = __find_governor(str_governor);
 420                        }
 421
 422                        kfree(name);
 423                }
 424
 425                if (t != NULL) {
 426                        *governor = t;
 427                        err = 0;
 428                }
 429
 430                mutex_unlock(&cpufreq_governor_mutex);
 431        }
 432  out:
 433        return err;
 434}
 435
 436
 437/* drivers/base/cpu.c */
 438extern struct sysdev_class cpu_sysdev_class;
 439
 440
 441/**
 442 * cpufreq_per_cpu_attr_read() / show_##file_name() -
 443 * print out cpufreq information
 444 *
 445 * Write out information from cpufreq_driver->policy[cpu]; object must be
 446 * "unsigned int".
 447 */
 448
 449#define show_one(file_name, object)                     \
 450static ssize_t show_##file_name                         \
 451(struct cpufreq_policy *policy, char *buf)              \
 452{                                                       \
 453        return sprintf (buf, "%u\n", policy->object);   \
 454}
 455
 456show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 457show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 458show_one(scaling_min_freq, min);
 459show_one(scaling_max_freq, max);
 460show_one(scaling_cur_freq, cur);
 461
 462static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                struct cpufreq_policy *policy);
 464
 465/**
 466 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467 */
 468#define store_one(file_name, object)                    \
 469static ssize_t store_##file_name                                        \
 470(struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471{                                                                       \
 472        unsigned int ret = -EINVAL;                                     \
 473        struct cpufreq_policy new_policy;                               \
 474                                                                        \
 475        ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476        if (ret)                                                        \
 477                return -EINVAL;                                         \
 478                                                                        \
 479        ret = sscanf (buf, "%u", &new_policy.object);                   \
 480        if (ret != 1)                                                   \
 481                return -EINVAL;                                         \
 482                                                                        \
 483        ret = __cpufreq_set_policy(policy, &new_policy);                \
 484        policy->user_policy.object = policy->object;                    \
 485                                                                        \
 486        return ret ? ret : count;                                       \
 487}
 488
 489store_one(scaling_min_freq,min);
 490store_one(scaling_max_freq,max);
 491
 492/**
 493 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494 */
 495static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                        char *buf)
 497{
 498        unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499        if (!cur_freq)
 500                return sprintf(buf, "<unknown>");
 501        return sprintf(buf, "%u\n", cur_freq);
 502}
 503
 504
 505/**
 506 * show_scaling_governor - show the current policy for the specified CPU
 507 */
 508static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 509{
 510        if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 511                return sprintf(buf, "powersave\n");
 512        else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 513                return sprintf(buf, "performance\n");
 514        else if (policy->governor)
 515                return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 516        return -EINVAL;
 517}
 518
 519
 520/**
 521 * store_scaling_governor - store policy for the specified CPU
 522 */
 523static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 524                                        const char *buf, size_t count)
 525{
 526        unsigned int ret = -EINVAL;
 527        char    str_governor[16];
 528        struct cpufreq_policy new_policy;
 529
 530        ret = cpufreq_get_policy(&new_policy, policy->cpu);
 531        if (ret)
 532                return ret;
 533
 534        ret = sscanf (buf, "%15s", str_governor);
 535        if (ret != 1)
 536                return -EINVAL;
 537
 538        if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 539                                                &new_policy.governor))
 540                return -EINVAL;
 541
 542        /* Do not use cpufreq_set_policy here or the user_policy.max
 543           will be wrongly overridden */
 544        ret = __cpufreq_set_policy(policy, &new_policy);
 545
 546        policy->user_policy.policy = policy->policy;
 547        policy->user_policy.governor = policy->governor;
 548
 549        if (ret)
 550                return ret;
 551        else
 552                return count;
 553}
 554
 555/**
 556 * show_scaling_driver - show the cpufreq driver currently loaded
 557 */
 558static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 559{
 560        return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 561}
 562
 563/**
 564 * show_scaling_available_governors - show the available CPUfreq governors
 565 */
 566static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 567                                                char *buf)
 568{
 569        ssize_t i = 0;
 570        struct cpufreq_governor *t;
 571
 572        if (!cpufreq_driver->target) {
 573                i += sprintf(buf, "performance powersave");
 574                goto out;
 575        }
 576
 577        list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 578                if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 579                        goto out;
 580                i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581        }
 582out:
 583        i += sprintf(&buf[i], "\n");
 584        return i;
 585}
 586
 587static ssize_t show_cpus(cpumask_t mask, char *buf)
 588{
 589        ssize_t i = 0;
 590        unsigned int cpu;
 591
 592        for_each_cpu_mask_nr(cpu, mask) {
 593                if (i)
 594                        i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 595                i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 596                if (i >= (PAGE_SIZE - 5))
 597                    break;
 598        }
 599        i += sprintf(&buf[i], "\n");
 600        return i;
 601}
 602
 603/**
 604 * show_related_cpus - show the CPUs affected by each transition even if
 605 * hw coordination is in use
 606 */
 607static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 608{
 609        if (cpus_empty(policy->related_cpus))
 610                return show_cpus(policy->cpus, buf);
 611        return show_cpus(policy->related_cpus, buf);
 612}
 613
 614/**
 615 * show_affected_cpus - show the CPUs affected by each transition
 616 */
 617static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618{
 619        return show_cpus(policy->cpus, buf);
 620}
 621
 622static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                        const char *buf, size_t count)
 624{
 625        unsigned int freq = 0;
 626        unsigned int ret;
 627
 628        if (!policy->governor || !policy->governor->store_setspeed)
 629                return -EINVAL;
 630
 631        ret = sscanf(buf, "%u", &freq);
 632        if (ret != 1)
 633                return -EINVAL;
 634
 635        policy->governor->store_setspeed(policy, freq);
 636
 637        return count;
 638}
 639
 640static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641{
 642        if (!policy->governor || !policy->governor->show_setspeed)
 643                return sprintf(buf, "<unsupported>\n");
 644
 645        return policy->governor->show_setspeed(policy, buf);
 646}
 647
 648#define define_one_ro(_name) \
 649static struct freq_attr _name = \
 650__ATTR(_name, 0444, show_##_name, NULL)
 651
 652#define define_one_ro0400(_name) \
 653static struct freq_attr _name = \
 654__ATTR(_name, 0400, show_##_name, NULL)
 655
 656#define define_one_rw(_name) \
 657static struct freq_attr _name = \
 658__ATTR(_name, 0644, show_##_name, store_##_name)
 659
 660define_one_ro0400(cpuinfo_cur_freq);
 661define_one_ro(cpuinfo_min_freq);
 662define_one_ro(cpuinfo_max_freq);
 663define_one_ro(scaling_available_governors);
 664define_one_ro(scaling_driver);
 665define_one_ro(scaling_cur_freq);
 666define_one_ro(related_cpus);
 667define_one_ro(affected_cpus);
 668define_one_rw(scaling_min_freq);
 669define_one_rw(scaling_max_freq);
 670define_one_rw(scaling_governor);
 671define_one_rw(scaling_setspeed);
 672
 673static struct attribute *default_attrs[] = {
 674        &cpuinfo_min_freq.attr,
 675        &cpuinfo_max_freq.attr,
 676        &scaling_min_freq.attr,
 677        &scaling_max_freq.attr,
 678        &affected_cpus.attr,
 679        &related_cpus.attr,
 680        &scaling_governor.attr,
 681        &scaling_driver.attr,
 682        &scaling_available_governors.attr,
 683        &scaling_setspeed.attr,
 684        NULL
 685};
 686
 687#define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 688#define to_attr(a) container_of(a,struct freq_attr,attr)
 689
 690static ssize_t show(struct kobject *kobj, struct attribute *attr ,char *buf)
 691{
 692        struct cpufreq_policy *policy = to_policy(kobj);
 693        struct freq_attr *fattr = to_attr(attr);
 694        ssize_t ret = -EINVAL;
 695        policy = cpufreq_cpu_get(policy->cpu);
 696        if (!policy)
 697                goto no_policy;
 698
 699        if (lock_policy_rwsem_read(policy->cpu) < 0)
 700                goto fail;
 701
 702        if (fattr->show)
 703                ret = fattr->show(policy, buf);
 704        else
 705                ret = -EIO;
 706
 707        unlock_policy_rwsem_read(policy->cpu);
 708fail:
 709        cpufreq_cpu_put(policy);
 710no_policy:
 711        return ret;
 712}
 713
 714static ssize_t store(struct kobject *kobj, struct attribute *attr,
 715                     const char *buf, size_t count)
 716{
 717        struct cpufreq_policy *policy = to_policy(kobj);
 718        struct freq_attr *fattr = to_attr(attr);
 719        ssize_t ret = -EINVAL;
 720        policy = cpufreq_cpu_get(policy->cpu);
 721        if (!policy)
 722                goto no_policy;
 723
 724        if (lock_policy_rwsem_write(policy->cpu) < 0)
 725                goto fail;
 726
 727        if (fattr->store)
 728                ret = fattr->store(policy, buf, count);
 729        else
 730                ret = -EIO;
 731
 732        unlock_policy_rwsem_write(policy->cpu);
 733fail:
 734        cpufreq_cpu_put(policy);
 735no_policy:
 736        return ret;
 737}
 738
 739static void cpufreq_sysfs_release(struct kobject *kobj)
 740{
 741        struct cpufreq_policy *policy = to_policy(kobj);
 742        dprintk("last reference is dropped\n");
 743        complete(&policy->kobj_unregister);
 744}
 745
 746static struct sysfs_ops sysfs_ops = {
 747        .show   = show,
 748        .store  = store,
 749};
 750
 751static struct kobj_type ktype_cpufreq = {
 752        .sysfs_ops      = &sysfs_ops,
 753        .default_attrs  = default_attrs,
 754        .release        = cpufreq_sysfs_release,
 755};
 756
 757
 758/**
 759 * cpufreq_add_dev - add a CPU device
 760 *
 761 * Adds the cpufreq interface for a CPU device.
 762 */
 763static int cpufreq_add_dev(struct sys_device *sys_dev)
 764{
 765        unsigned int cpu = sys_dev->id;
 766        int ret = 0;
 767        struct cpufreq_policy new_policy;
 768        struct cpufreq_policy *policy;
 769        struct freq_attr **drv_attr;
 770        struct sys_device *cpu_sys_dev;
 771        unsigned long flags;
 772        unsigned int j;
 773#ifdef CONFIG_SMP
 774        struct cpufreq_policy *managed_policy;
 775#endif
 776
 777        if (cpu_is_offline(cpu))
 778                return 0;
 779
 780        cpufreq_debug_disable_ratelimit();
 781        dprintk("adding CPU %u\n", cpu);
 782
 783#ifdef CONFIG_SMP
 784        /* check whether a different CPU already registered this
 785         * CPU because it is in the same boat. */
 786        policy = cpufreq_cpu_get(cpu);
 787        if (unlikely(policy)) {
 788                cpufreq_cpu_put(policy);
 789                cpufreq_debug_enable_ratelimit();
 790                return 0;
 791        }
 792#endif
 793
 794        if (!try_module_get(cpufreq_driver->owner)) {
 795                ret = -EINVAL;
 796                goto module_out;
 797        }
 798
 799        policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 800        if (!policy) {
 801                ret = -ENOMEM;
 802                goto nomem_out;
 803        }
 804
 805        policy->cpu = cpu;
 806        policy->cpus = cpumask_of_cpu(cpu);
 807
 808        /* Initially set CPU itself as the policy_cpu */
 809        per_cpu(policy_cpu, cpu) = cpu;
 810        lock_policy_rwsem_write(cpu);
 811
 812        init_completion(&policy->kobj_unregister);
 813        INIT_WORK(&policy->update, handle_update);
 814
 815        /* Set governor before ->init, so that driver could check it */
 816        policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 817        /* call driver. From then on the cpufreq must be able
 818         * to accept all calls to ->verify and ->setpolicy for this CPU
 819         */
 820        ret = cpufreq_driver->init(policy);
 821        if (ret) {
 822                dprintk("initialization failed\n");
 823                goto err_out;
 824        }
 825        policy->user_policy.min = policy->cpuinfo.min_freq;
 826        policy->user_policy.max = policy->cpuinfo.max_freq;
 827
 828        blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 829                                     CPUFREQ_START, policy);
 830
 831#ifdef CONFIG_SMP
 832
 833#ifdef CONFIG_HOTPLUG_CPU
 834        if (per_cpu(cpufreq_cpu_governor, cpu)) {
 835                policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 836                dprintk("Restoring governor %s for cpu %d\n",
 837                       policy->governor->name, cpu);
 838        }
 839#endif
 840
 841        for_each_cpu_mask_nr(j, policy->cpus) {
 842                if (cpu == j)
 843                        continue;
 844
 845                /* check for existing affected CPUs.  They may not be aware
 846                 * of it due to CPU Hotplug.
 847                 */
 848                managed_policy = cpufreq_cpu_get(j);            // FIXME: Where is this released?  What about error paths?
 849                if (unlikely(managed_policy)) {
 850
 851                        /* Set proper policy_cpu */
 852                        unlock_policy_rwsem_write(cpu);
 853                        per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 854
 855                        if (lock_policy_rwsem_write(cpu) < 0)
 856                                goto err_out_driver_exit;
 857
 858                        spin_lock_irqsave(&cpufreq_driver_lock, flags);
 859                        managed_policy->cpus = policy->cpus;
 860                        per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 861                        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 862
 863                        dprintk("CPU already managed, adding link\n");
 864                        ret = sysfs_create_link(&sys_dev->kobj,
 865                                                &managed_policy->kobj,
 866                                                "cpufreq");
 867                        if (ret)
 868                                goto err_out_driver_exit;
 869
 870                        cpufreq_debug_enable_ratelimit();
 871                        ret = 0;
 872                        goto err_out_driver_exit; /* call driver->exit() */
 873                }
 874        }
 875#endif
 876        memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 877
 878        /* prepare interface data */
 879        ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 880                                   "cpufreq");
 881        if (ret)
 882                goto err_out_driver_exit;
 883
 884        /* set up files for this cpu device */
 885        drv_attr = cpufreq_driver->attr;
 886        while ((drv_attr) && (*drv_attr)) {
 887                ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 888                if (ret)
 889                        goto err_out_driver_exit;
 890                drv_attr++;
 891        }
 892        if (cpufreq_driver->get) {
 893                ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 894                if (ret)
 895                        goto err_out_driver_exit;
 896        }
 897        if (cpufreq_driver->target) {
 898                ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 899                if (ret)
 900                        goto err_out_driver_exit;
 901        }
 902
 903        spin_lock_irqsave(&cpufreq_driver_lock, flags);
 904        for_each_cpu_mask_nr(j, policy->cpus) {
 905                per_cpu(cpufreq_cpu_data, j) = policy;
 906                per_cpu(policy_cpu, j) = policy->cpu;
 907        }
 908        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 909
 910        /* symlink affected CPUs */
 911        for_each_cpu_mask_nr(j, policy->cpus) {
 912                if (j == cpu)
 913                        continue;
 914                if (!cpu_online(j))
 915                        continue;
 916
 917                dprintk("CPU %u already managed, adding link\n", j);
 918                cpufreq_cpu_get(cpu);
 919                cpu_sys_dev = get_cpu_sysdev(j);
 920                ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 921                                        "cpufreq");
 922                if (ret)
 923                        goto err_out_unregister;
 924        }
 925
 926        policy->governor = NULL; /* to assure that the starting sequence is
 927                                  * run in cpufreq_set_policy */
 928
 929        /* set default policy */
 930        ret = __cpufreq_set_policy(policy, &new_policy);
 931        policy->user_policy.policy = policy->policy;
 932        policy->user_policy.governor = policy->governor;
 933
 934        if (ret) {
 935                dprintk("setting policy failed\n");
 936                goto err_out_unregister;
 937        }
 938
 939        unlock_policy_rwsem_write(cpu);
 940
 941        kobject_uevent(&policy->kobj, KOBJ_ADD);
 942        module_put(cpufreq_driver->owner);
 943        dprintk("initialization complete\n");
 944        cpufreq_debug_enable_ratelimit();
 945
 946        return 0;
 947
 948
 949err_out_unregister:
 950        spin_lock_irqsave(&cpufreq_driver_lock, flags);
 951        for_each_cpu_mask_nr(j, policy->cpus)
 952                per_cpu(cpufreq_cpu_data, j) = NULL;
 953        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 954
 955        kobject_put(&policy->kobj);
 956        wait_for_completion(&policy->kobj_unregister);
 957
 958err_out_driver_exit:
 959        if (cpufreq_driver->exit)
 960                cpufreq_driver->exit(policy);
 961
 962err_out:
 963        unlock_policy_rwsem_write(cpu);
 964        kfree(policy);
 965
 966nomem_out:
 967        module_put(cpufreq_driver->owner);
 968module_out:
 969        cpufreq_debug_enable_ratelimit();
 970        return ret;
 971}
 972
 973
 974/**
 975 * __cpufreq_remove_dev - remove a CPU device
 976 *
 977 * Removes the cpufreq interface for a CPU device.
 978 * Caller should already have policy_rwsem in write mode for this CPU.
 979 * This routine frees the rwsem before returning.
 980 */
 981static int __cpufreq_remove_dev(struct sys_device *sys_dev)
 982{
 983        unsigned int cpu = sys_dev->id;
 984        unsigned long flags;
 985        struct cpufreq_policy *data;
 986#ifdef CONFIG_SMP
 987        struct sys_device *cpu_sys_dev;
 988        unsigned int j;
 989#endif
 990
 991        cpufreq_debug_disable_ratelimit();
 992        dprintk("unregistering CPU %u\n", cpu);
 993
 994        spin_lock_irqsave(&cpufreq_driver_lock, flags);
 995        data = per_cpu(cpufreq_cpu_data, cpu);
 996
 997        if (!data) {
 998                spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 999                cpufreq_debug_enable_ratelimit();
1000                unlock_policy_rwsem_write(cpu);
1001                return -EINVAL;
1002        }
1003        per_cpu(cpufreq_cpu_data, cpu) = NULL;
1004
1005
1006#ifdef CONFIG_SMP
1007        /* if this isn't the CPU which is the parent of the kobj, we
1008         * only need to unlink, put and exit
1009         */
1010        if (unlikely(cpu != data->cpu)) {
1011                dprintk("removing link\n");
1012                cpu_clear(cpu, data->cpus);
1013                spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1014                sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1015                cpufreq_cpu_put(data);
1016                cpufreq_debug_enable_ratelimit();
1017                unlock_policy_rwsem_write(cpu);
1018                return 0;
1019        }
1020#endif
1021
1022#ifdef CONFIG_SMP
1023
1024#ifdef CONFIG_HOTPLUG_CPU
1025        per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1026#endif
1027
1028        /* if we have other CPUs still registered, we need to unlink them,
1029         * or else wait_for_completion below will lock up. Clean the
1030         * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1031         * the sysfs links afterwards.
1032         */
1033        if (unlikely(cpus_weight(data->cpus) > 1)) {
1034                for_each_cpu_mask_nr(j, data->cpus) {
1035                        if (j == cpu)
1036                                continue;
1037                        per_cpu(cpufreq_cpu_data, j) = NULL;
1038                }
1039        }
1040
1041        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1042
1043        if (unlikely(cpus_weight(data->cpus) > 1)) {
1044                for_each_cpu_mask_nr(j, data->cpus) {
1045                        if (j == cpu)
1046                                continue;
1047                        dprintk("removing link for cpu %u\n", j);
1048#ifdef CONFIG_HOTPLUG_CPU
1049                        per_cpu(cpufreq_cpu_governor, j) = data->governor;
1050#endif
1051                        cpu_sys_dev = get_cpu_sysdev(j);
1052                        sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1053                        cpufreq_cpu_put(data);
1054                }
1055        }
1056#else
1057        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1058#endif
1059
1060        if (cpufreq_driver->target)
1061                __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1062
1063        unlock_policy_rwsem_write(cpu);
1064
1065        kobject_put(&data->kobj);
1066
1067        /* we need to make sure that the underlying kobj is actually
1068         * not referenced anymore by anybody before we proceed with
1069         * unloading.
1070         */
1071        dprintk("waiting for dropping of refcount\n");
1072        wait_for_completion(&data->kobj_unregister);
1073        dprintk("wait complete\n");
1074
1075        if (cpufreq_driver->exit)
1076                cpufreq_driver->exit(data);
1077
1078        kfree(data);
1079
1080        cpufreq_debug_enable_ratelimit();
1081        return 0;
1082}
1083
1084
1085static int cpufreq_remove_dev(struct sys_device *sys_dev)
1086{
1087        unsigned int cpu = sys_dev->id;
1088        int retval;
1089
1090        if (cpu_is_offline(cpu))
1091                return 0;
1092
1093        if (unlikely(lock_policy_rwsem_write(cpu)))
1094                BUG();
1095
1096        retval = __cpufreq_remove_dev(sys_dev);
1097        return retval;
1098}
1099
1100
1101static void handle_update(struct work_struct *work)
1102{
1103        struct cpufreq_policy *policy =
1104                container_of(work, struct cpufreq_policy, update);
1105        unsigned int cpu = policy->cpu;
1106        dprintk("handle_update for cpu %u called\n", cpu);
1107        cpufreq_update_policy(cpu);
1108}
1109
1110/**
1111 *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1112 *      @cpu: cpu number
1113 *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1114 *      @new_freq: CPU frequency the CPU actually runs at
1115 *
1116 *      We adjust to current frequency first, and need to clean up later. So either call
1117 *      to cpufreq_update_policy() or schedule handle_update()).
1118 */
1119static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1120                                unsigned int new_freq)
1121{
1122        struct cpufreq_freqs freqs;
1123
1124        dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1125               "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1126
1127        freqs.cpu = cpu;
1128        freqs.old = old_freq;
1129        freqs.new = new_freq;
1130        cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1131        cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1132}
1133
1134
1135/**
1136 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1137 * @cpu: CPU number
1138 *
1139 * This is the last known freq, without actually getting it from the driver.
1140 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1141 */
1142unsigned int cpufreq_quick_get(unsigned int cpu)
1143{
1144        struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1145        unsigned int ret_freq = 0;
1146
1147        if (policy) {
1148                ret_freq = policy->cur;
1149                cpufreq_cpu_put(policy);
1150        }
1151
1152        return ret_freq;
1153}
1154EXPORT_SYMBOL(cpufreq_quick_get);
1155
1156
1157static unsigned int __cpufreq_get(unsigned int cpu)
1158{
1159        struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1160        unsigned int ret_freq = 0;
1161
1162        if (!cpufreq_driver->get)
1163                return ret_freq;
1164
1165        ret_freq = cpufreq_driver->get(cpu);
1166
1167        if (ret_freq && policy->cur &&
1168                !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1169                /* verify no discrepancy between actual and
1170                                        saved value exists */
1171                if (unlikely(ret_freq != policy->cur)) {
1172                        cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1173                        schedule_work(&policy->update);
1174                }
1175        }
1176
1177        return ret_freq;
1178}
1179
1180/**
1181 * cpufreq_get - get the current CPU frequency (in kHz)
1182 * @cpu: CPU number
1183 *
1184 * Get the CPU current (static) CPU frequency
1185 */
1186unsigned int cpufreq_get(unsigned int cpu)
1187{
1188        unsigned int ret_freq = 0;
1189        struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1190
1191        if (!policy)
1192                goto out;
1193
1194        if (unlikely(lock_policy_rwsem_read(cpu)))
1195                goto out_policy;
1196
1197        ret_freq = __cpufreq_get(cpu);
1198
1199        unlock_policy_rwsem_read(cpu);
1200
1201out_policy:
1202        cpufreq_cpu_put(policy);
1203out:
1204        return ret_freq;
1205}
1206EXPORT_SYMBOL(cpufreq_get);
1207
1208
1209/**
1210 *      cpufreq_suspend - let the low level driver prepare for suspend
1211 */
1212
1213static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1214{
1215        int cpu = sysdev->id;
1216        int ret = 0;
1217        unsigned int cur_freq = 0;
1218        struct cpufreq_policy *cpu_policy;
1219
1220        dprintk("suspending cpu %u\n", cpu);
1221
1222        if (!cpu_online(cpu))
1223                return 0;
1224
1225        /* we may be lax here as interrupts are off. Nonetheless
1226         * we need to grab the correct cpu policy, as to check
1227         * whether we really run on this CPU.
1228         */
1229
1230        cpu_policy = cpufreq_cpu_get(cpu);
1231        if (!cpu_policy)
1232                return -EINVAL;
1233
1234        /* only handle each CPU group once */
1235        if (unlikely(cpu_policy->cpu != cpu))
1236                goto out;
1237
1238        if (cpufreq_driver->suspend) {
1239                ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1240                if (ret) {
1241                        printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1242                                        "step on CPU %u\n", cpu_policy->cpu);
1243                        goto out;
1244                }
1245        }
1246
1247        if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1248                goto out;
1249
1250        if (cpufreq_driver->get)
1251                cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1252
1253        if (!cur_freq || !cpu_policy->cur) {
1254                printk(KERN_ERR "cpufreq: suspend failed to assert current "
1255                       "frequency is what timing core thinks it is.\n");
1256                goto out;
1257        }
1258
1259        if (unlikely(cur_freq != cpu_policy->cur)) {
1260                struct cpufreq_freqs freqs;
1261
1262                if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1263                        dprintk("Warning: CPU frequency is %u, "
1264                               "cpufreq assumed %u kHz.\n",
1265                               cur_freq, cpu_policy->cur);
1266
1267                freqs.cpu = cpu;
1268                freqs.old = cpu_policy->cur;
1269                freqs.new = cur_freq;
1270
1271                srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1272                                    CPUFREQ_SUSPENDCHANGE, &freqs);
1273                adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1274
1275                cpu_policy->cur = cur_freq;
1276        }
1277
1278out:
1279        cpufreq_cpu_put(cpu_policy);
1280        return ret;
1281}
1282
1283/**
1284 *      cpufreq_resume -  restore proper CPU frequency handling after resume
1285 *
1286 *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1287 *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1288 *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1289 *          restored.
1290 */
1291static int cpufreq_resume(struct sys_device *sysdev)
1292{
1293        int cpu = sysdev->id;
1294        int ret = 0;
1295        struct cpufreq_policy *cpu_policy;
1296
1297        dprintk("resuming cpu %u\n", cpu);
1298
1299        if (!cpu_online(cpu))
1300                return 0;
1301
1302        /* we may be lax here as interrupts are off. Nonetheless
1303         * we need to grab the correct cpu policy, as to check
1304         * whether we really run on this CPU.
1305         */
1306
1307        cpu_policy = cpufreq_cpu_get(cpu);
1308        if (!cpu_policy)
1309                return -EINVAL;
1310
1311        /* only handle each CPU group once */
1312        if (unlikely(cpu_policy->cpu != cpu))
1313                goto fail;
1314
1315        if (cpufreq_driver->resume) {
1316                ret = cpufreq_driver->resume(cpu_policy);
1317                if (ret) {
1318                        printk(KERN_ERR "cpufreq: resume failed in ->resume "
1319                                        "step on CPU %u\n", cpu_policy->cpu);
1320                        goto fail;
1321                }
1322        }
1323
1324        if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1325                unsigned int cur_freq = 0;
1326
1327                if (cpufreq_driver->get)
1328                        cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1329
1330                if (!cur_freq || !cpu_policy->cur) {
1331                        printk(KERN_ERR "cpufreq: resume failed to assert "
1332                                        "current frequency is what timing core "
1333                                        "thinks it is.\n");
1334                        goto out;
1335                }
1336
1337                if (unlikely(cur_freq != cpu_policy->cur)) {
1338                        struct cpufreq_freqs freqs;
1339
1340                        if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1341                                dprintk("Warning: CPU frequency "
1342                                       "is %u, cpufreq assumed %u kHz.\n",
1343                                       cur_freq, cpu_policy->cur);
1344
1345                        freqs.cpu = cpu;
1346                        freqs.old = cpu_policy->cur;
1347                        freqs.new = cur_freq;
1348
1349                        srcu_notifier_call_chain(
1350                                        &cpufreq_transition_notifier_list,
1351                                        CPUFREQ_RESUMECHANGE, &freqs);
1352                        adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1353
1354                        cpu_policy->cur = cur_freq;
1355                }
1356        }
1357
1358out:
1359        schedule_work(&cpu_policy->update);
1360fail:
1361        cpufreq_cpu_put(cpu_policy);
1362        return ret;
1363}
1364
1365static struct sysdev_driver cpufreq_sysdev_driver = {
1366        .add            = cpufreq_add_dev,
1367        .remove         = cpufreq_remove_dev,
1368        .suspend        = cpufreq_suspend,
1369        .resume         = cpufreq_resume,
1370};
1371
1372
1373/*********************************************************************
1374 *                     NOTIFIER LISTS INTERFACE                      *
1375 *********************************************************************/
1376
1377/**
1378 *      cpufreq_register_notifier - register a driver with cpufreq
1379 *      @nb: notifier function to register
1380 *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1381 *
1382 *      Add a driver to one of two lists: either a list of drivers that
1383 *      are notified about clock rate changes (once before and once after
1384 *      the transition), or a list of drivers that are notified about
1385 *      changes in cpufreq policy.
1386 *
1387 *      This function may sleep, and has the same return conditions as
1388 *      blocking_notifier_chain_register.
1389 */
1390int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1391{
1392        int ret;
1393
1394        WARN_ON(!init_cpufreq_transition_notifier_list_called);
1395
1396        switch (list) {
1397        case CPUFREQ_TRANSITION_NOTIFIER:
1398                ret = srcu_notifier_chain_register(
1399                                &cpufreq_transition_notifier_list, nb);
1400                break;
1401        case CPUFREQ_POLICY_NOTIFIER:
1402                ret = blocking_notifier_chain_register(
1403                                &cpufreq_policy_notifier_list, nb);
1404                break;
1405        default:
1406                ret = -EINVAL;
1407        }
1408
1409        return ret;
1410}
1411EXPORT_SYMBOL(cpufreq_register_notifier);
1412
1413
1414/**
1415 *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1416 *      @nb: notifier block to be unregistered
1417 *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1418 *
1419 *      Remove a driver from the CPU frequency notifier list.
1420 *
1421 *      This function may sleep, and has the same return conditions as
1422 *      blocking_notifier_chain_unregister.
1423 */
1424int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1425{
1426        int ret;
1427
1428        switch (list) {
1429        case CPUFREQ_TRANSITION_NOTIFIER:
1430                ret = srcu_notifier_chain_unregister(
1431                                &cpufreq_transition_notifier_list, nb);
1432                break;
1433        case CPUFREQ_POLICY_NOTIFIER:
1434                ret = blocking_notifier_chain_unregister(
1435                                &cpufreq_policy_notifier_list, nb);
1436                break;
1437        default:
1438                ret = -EINVAL;
1439        }
1440
1441        return ret;
1442}
1443EXPORT_SYMBOL(cpufreq_unregister_notifier);
1444
1445
1446/*********************************************************************
1447 *                              GOVERNORS                            *
1448 *********************************************************************/
1449
1450
1451int __cpufreq_driver_target(struct cpufreq_policy *policy,
1452                            unsigned int target_freq,
1453                            unsigned int relation)
1454{
1455        int retval = -EINVAL;
1456
1457        dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1458                target_freq, relation);
1459        if (cpu_online(policy->cpu) && cpufreq_driver->target)
1460                retval = cpufreq_driver->target(policy, target_freq, relation);
1461
1462        return retval;
1463}
1464EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1465
1466int cpufreq_driver_target(struct cpufreq_policy *policy,
1467                          unsigned int target_freq,
1468                          unsigned int relation)
1469{
1470        int ret = -EINVAL;
1471
1472        policy = cpufreq_cpu_get(policy->cpu);
1473        if (!policy)
1474                goto no_policy;
1475
1476        if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1477                goto fail;
1478
1479        ret = __cpufreq_driver_target(policy, target_freq, relation);
1480
1481        unlock_policy_rwsem_write(policy->cpu);
1482
1483fail:
1484        cpufreq_cpu_put(policy);
1485no_policy:
1486        return ret;
1487}
1488EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1489
1490int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1491{
1492        int ret = 0;
1493
1494        policy = cpufreq_cpu_get(policy->cpu);
1495        if (!policy)
1496                return -EINVAL;
1497
1498        if (cpu_online(cpu) && cpufreq_driver->getavg)
1499                ret = cpufreq_driver->getavg(policy, cpu);
1500
1501        cpufreq_cpu_put(policy);
1502        return ret;
1503}
1504EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1505
1506/*
1507 * when "event" is CPUFREQ_GOV_LIMITS
1508 */
1509
1510static int __cpufreq_governor(struct cpufreq_policy *policy,
1511                                        unsigned int event)
1512{
1513        int ret;
1514
1515        /* Only must be defined when default governor is known to have latency
1516           restrictions, like e.g. conservative or ondemand.
1517           That this is the case is already ensured in Kconfig
1518        */
1519#ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1520        struct cpufreq_governor *gov = &cpufreq_gov_performance;
1521#else
1522        struct cpufreq_governor *gov = NULL;
1523#endif
1524
1525        if (policy->governor->max_transition_latency &&
1526            policy->cpuinfo.transition_latency >
1527            policy->governor->max_transition_latency) {
1528                if (!gov)
1529                        return -EINVAL;
1530                else {
1531                        printk(KERN_WARNING "%s governor failed, too long"
1532                               " transition latency of HW, fallback"
1533                               " to %s governor\n",
1534                               policy->governor->name,
1535                               gov->name);
1536                        policy->governor = gov;
1537                }
1538        }
1539
1540        if (!try_module_get(policy->governor->owner))
1541                return -EINVAL;
1542
1543        dprintk("__cpufreq_governor for CPU %u, event %u\n",
1544                                                policy->cpu, event);
1545        ret = policy->governor->governor(policy, event);
1546
1547        /* we keep one module reference alive for
1548                        each CPU governed by this CPU */
1549        if ((event != CPUFREQ_GOV_START) || ret)
1550                module_put(policy->governor->owner);
1551        if ((event == CPUFREQ_GOV_STOP) && !ret)
1552                module_put(policy->governor->owner);
1553
1554        return ret;
1555}
1556
1557
1558int cpufreq_register_governor(struct cpufreq_governor *governor)
1559{
1560        int err;
1561
1562        if (!governor)
1563                return -EINVAL;
1564
1565        mutex_lock(&cpufreq_governor_mutex);
1566
1567        err = -EBUSY;
1568        if (__find_governor(governor->name) == NULL) {
1569                err = 0;
1570                list_add(&governor->governor_list, &cpufreq_governor_list);
1571        }
1572
1573        mutex_unlock(&cpufreq_governor_mutex);
1574        return err;
1575}
1576EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1577
1578
1579void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1580{
1581        if (!governor)
1582                return;
1583
1584        mutex_lock(&cpufreq_governor_mutex);
1585        list_del(&governor->governor_list);
1586        mutex_unlock(&cpufreq_governor_mutex);
1587        return;
1588}
1589EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1590
1591
1592
1593/*********************************************************************
1594 *                          POLICY INTERFACE                         *
1595 *********************************************************************/
1596
1597/**
1598 * cpufreq_get_policy - get the current cpufreq_policy
1599 * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1600 *
1601 * Reads the current cpufreq policy.
1602 */
1603int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1604{
1605        struct cpufreq_policy *cpu_policy;
1606        if (!policy)
1607                return -EINVAL;
1608
1609        cpu_policy = cpufreq_cpu_get(cpu);
1610        if (!cpu_policy)
1611                return -EINVAL;
1612
1613        memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1614
1615        cpufreq_cpu_put(cpu_policy);
1616        return 0;
1617}
1618EXPORT_SYMBOL(cpufreq_get_policy);
1619
1620
1621/*
1622 * data   : current policy.
1623 * policy : policy to be set.
1624 */
1625static int __cpufreq_set_policy(struct cpufreq_policy *data,
1626                                struct cpufreq_policy *policy)
1627{
1628        int ret = 0;
1629
1630        cpufreq_debug_disable_ratelimit();
1631        dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1632                policy->min, policy->max);
1633
1634        memcpy(&policy->cpuinfo, &data->cpuinfo,
1635                                sizeof(struct cpufreq_cpuinfo));
1636
1637        if (policy->min > data->max || policy->max < data->min) {
1638                ret = -EINVAL;
1639                goto error_out;
1640        }
1641
1642        /* verify the cpu speed can be set within this limit */
1643        ret = cpufreq_driver->verify(policy);
1644        if (ret)
1645                goto error_out;
1646
1647        /* adjust if necessary - all reasons */
1648        blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1649                        CPUFREQ_ADJUST, policy);
1650
1651        /* adjust if necessary - hardware incompatibility*/
1652        blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1653                        CPUFREQ_INCOMPATIBLE, policy);
1654
1655        /* verify the cpu speed can be set within this limit,
1656           which might be different to the first one */
1657        ret = cpufreq_driver->verify(policy);
1658        if (ret)
1659                goto error_out;
1660
1661        /* notification of the new policy */
1662        blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1663                        CPUFREQ_NOTIFY, policy);
1664
1665        data->min = policy->min;
1666        data->max = policy->max;
1667
1668        dprintk("new min and max freqs are %u - %u kHz\n",
1669                                        data->min, data->max);
1670
1671        if (cpufreq_driver->setpolicy) {
1672                data->policy = policy->policy;
1673                dprintk("setting range\n");
1674                ret = cpufreq_driver->setpolicy(policy);
1675        } else {
1676                if (policy->governor != data->governor) {
1677                        /* save old, working values */
1678                        struct cpufreq_governor *old_gov = data->governor;
1679
1680                        dprintk("governor switch\n");
1681
1682                        /* end old governor */
1683                        if (data->governor)
1684                                __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1685
1686                        /* start new governor */
1687                        data->governor = policy->governor;
1688                        if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1689                                /* new governor failed, so re-start old one */
1690                                dprintk("starting governor %s failed\n",
1691                                                        data->governor->name);
1692                                if (old_gov) {
1693                                        data->governor = old_gov;
1694                                        __cpufreq_governor(data,
1695                                                           CPUFREQ_GOV_START);
1696                                }
1697                                ret = -EINVAL;
1698                                goto error_out;
1699                        }
1700                        /* might be a policy change, too, so fall through */
1701                }
1702                dprintk("governor: change or update limits\n");
1703                __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1704        }
1705
1706error_out:
1707        cpufreq_debug_enable_ratelimit();
1708        return ret;
1709}
1710
1711/**
1712 *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1713 *      @cpu: CPU which shall be re-evaluated
1714 *
1715 *      Usefull for policy notifiers which have different necessities
1716 *      at different times.
1717 */
1718int cpufreq_update_policy(unsigned int cpu)
1719{
1720        struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1721        struct cpufreq_policy policy;
1722        int ret;
1723
1724        if (!data) {
1725                ret = -ENODEV;
1726                goto no_policy;
1727        }
1728
1729        if (unlikely(lock_policy_rwsem_write(cpu))) {
1730                ret = -EINVAL;
1731                goto fail;
1732        }
1733
1734        dprintk("updating policy for CPU %u\n", cpu);
1735        memcpy(&policy, data, sizeof(struct cpufreq_policy));
1736        policy.min = data->user_policy.min;
1737        policy.max = data->user_policy.max;
1738        policy.policy = data->user_policy.policy;
1739        policy.governor = data->user_policy.governor;
1740
1741        /* BIOS might change freq behind our back
1742          -> ask driver for current freq and notify governors about a change */
1743        if (cpufreq_driver->get) {
1744                policy.cur = cpufreq_driver->get(cpu);
1745                if (!data->cur) {
1746                        dprintk("Driver did not initialize current freq");
1747                        data->cur = policy.cur;
1748                } else {
1749                        if (data->cur != policy.cur)
1750                                cpufreq_out_of_sync(cpu, data->cur,
1751                                                                policy.cur);
1752                }
1753        }
1754
1755        ret = __cpufreq_set_policy(data, &policy);
1756
1757        unlock_policy_rwsem_write(cpu);
1758
1759fail:
1760        cpufreq_cpu_put(data);
1761no_policy:
1762        return ret;
1763}
1764EXPORT_SYMBOL(cpufreq_update_policy);
1765
1766static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1767                                        unsigned long action, void *hcpu)
1768{
1769        unsigned int cpu = (unsigned long)hcpu;
1770        struct sys_device *sys_dev;
1771
1772        sys_dev = get_cpu_sysdev(cpu);
1773        if (sys_dev) {
1774                switch (action) {
1775                case CPU_ONLINE:
1776                case CPU_ONLINE_FROZEN:
1777                        cpufreq_add_dev(sys_dev);
1778                        break;
1779                case CPU_DOWN_PREPARE:
1780                case CPU_DOWN_PREPARE_FROZEN:
1781                        if (unlikely(lock_policy_rwsem_write(cpu)))
1782                                BUG();
1783
1784                        __cpufreq_remove_dev(sys_dev);
1785                        break;
1786                case CPU_DOWN_FAILED:
1787                case CPU_DOWN_FAILED_FROZEN:
1788                        cpufreq_add_dev(sys_dev);
1789                        break;
1790                }
1791        }
1792        return NOTIFY_OK;
1793}
1794
1795static struct notifier_block __refdata cpufreq_cpu_notifier =
1796{
1797    .notifier_call = cpufreq_cpu_callback,
1798};
1799
1800/*********************************************************************
1801 *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1802 *********************************************************************/
1803
1804/**
1805 * cpufreq_register_driver - register a CPU Frequency driver
1806 * @driver_data: A struct cpufreq_driver containing the values#
1807 * submitted by the CPU Frequency driver.
1808 *
1809 *   Registers a CPU Frequency driver to this core code. This code
1810 * returns zero on success, -EBUSY when another driver got here first
1811 * (and isn't unregistered in the meantime).
1812 *
1813 */
1814int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1815{
1816        unsigned long flags;
1817        int ret;
1818
1819        if (!driver_data || !driver_data->verify || !driver_data->init ||
1820            ((!driver_data->setpolicy) && (!driver_data->target)))
1821                return -EINVAL;
1822
1823        dprintk("trying to register driver %s\n", driver_data->name);
1824
1825        if (driver_data->setpolicy)
1826                driver_data->flags |= CPUFREQ_CONST_LOOPS;
1827
1828        spin_lock_irqsave(&cpufreq_driver_lock, flags);
1829        if (cpufreq_driver) {
1830                spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1831                return -EBUSY;
1832        }
1833        cpufreq_driver = driver_data;
1834        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1835
1836        ret = sysdev_driver_register(&cpu_sysdev_class,
1837                                        &cpufreq_sysdev_driver);
1838
1839        if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1840                int i;
1841                ret = -ENODEV;
1842
1843                /* check for at least one working CPU */
1844                for (i = 0; i < nr_cpu_ids; i++)
1845                        if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1846                                ret = 0;
1847                                break;
1848                        }
1849
1850                /* if all ->init() calls failed, unregister */
1851                if (ret) {
1852                        dprintk("no CPU initialized for driver %s\n",
1853                                                        driver_data->name);
1854                        sysdev_driver_unregister(&cpu_sysdev_class,
1855                                                &cpufreq_sysdev_driver);
1856
1857                        spin_lock_irqsave(&cpufreq_driver_lock, flags);
1858                        cpufreq_driver = NULL;
1859                        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1860                }
1861        }
1862
1863        if (!ret) {
1864                register_hotcpu_notifier(&cpufreq_cpu_notifier);
1865                dprintk("driver %s up and running\n", driver_data->name);
1866                cpufreq_debug_enable_ratelimit();
1867        }
1868
1869        return ret;
1870}
1871EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1872
1873
1874/**
1875 * cpufreq_unregister_driver - unregister the current CPUFreq driver
1876 *
1877 *    Unregister the current CPUFreq driver. Only call this if you have
1878 * the right to do so, i.e. if you have succeeded in initialising before!
1879 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1880 * currently not initialised.
1881 */
1882int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1883{
1884        unsigned long flags;
1885
1886        cpufreq_debug_disable_ratelimit();
1887
1888        if (!cpufreq_driver || (driver != cpufreq_driver)) {
1889                cpufreq_debug_enable_ratelimit();
1890                return -EINVAL;
1891        }
1892
1893        dprintk("unregistering driver %s\n", driver->name);
1894
1895        sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1896        unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1897
1898        spin_lock_irqsave(&cpufreq_driver_lock, flags);
1899        cpufreq_driver = NULL;
1900        spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1901
1902        return 0;
1903}
1904EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1905
1906static int __init cpufreq_core_init(void)
1907{
1908        int cpu;
1909
1910        for_each_possible_cpu(cpu) {
1911                per_cpu(policy_cpu, cpu) = -1;
1912                init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1913        }
1914        return 0;
1915}
1916
1917core_initcall(cpufreq_core_init);
1918