drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy,
 108                 unsigned int event);
 109 static unsigned int __cpufreq_get(unsigned int cpu);
 110 static void handle_update(struct work_struct *work);
 111
 112 /**
 113  * Two notifier lists: the "policy" list is involved in the
 114  * validation process for a new CPU frequency policy; the
 115  * "transition" list for kernel code that needs to handle
 116  * changes to devices when the CPU clock speed changes.
 117  * The mutex locks both lists.
 118  */
 119 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 120 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 121
 122 static bool init_cpufreq_transition_notifier_list_called;
 123 static int __init init_cpufreq_transition_notifier_list(void)
 124 {
 125         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 126         init_cpufreq_transition_notifier_list_called = true;
 127         return 0;
 128 }
 129 pure_initcall(init_cpufreq_transition_notifier_list);
 130
 131 static LIST_HEAD(cpufreq_governor_list);
 132 static DEFINE_MUTEX(cpufreq_governor_mutex);
 133
 134 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 135 {
 136         struct cpufreq_policy *data;
 137         unsigned long flags;
 138
 139         if (cpu >= nr_cpu_ids)
 140                 goto err_out;
 141
 142         /* get the cpufreq driver */
 143         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 144
 145         if (!cpufreq_driver)
 146                 goto err_out_unlock;
 147
 148         if (!try_module_get(cpufreq_driver->owner))
 149                 goto err_out_unlock;
 150
 151
 152         /* get the CPU */
 153         data = per_cpu(cpufreq_cpu_data, cpu);
 154
 155         if (!data)
 156                 goto err_out_put_module;
 157
 158         if (!kobject_get(&data->kobj))
 159                 goto err_out_put_module;
 160
 161         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 162         return data;
 163
 164 err_out_put_module:
 165         module_put(cpufreq_driver->owner);
 166 err_out_unlock:
 167         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 168 err_out:
 169         return NULL;
 170 }
 171 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 172
 173
 174 void cpufreq_cpu_put(struct cpufreq_policy *data)
 175 {
 176         kobject_put(&data->kobj);
 177         module_put(cpufreq_driver->owner);
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 180
 181
 182 /*********************************************************************
 183  *                     UNIFIED DEBUG HELPERS                         *
 184  *********************************************************************/
 185 #ifdef CONFIG_CPU_FREQ_DEBUG
 186
 187 /* what part(s) of the CPUfreq subsystem are debugged? */
 188 static unsigned int debug;
 189
 190 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 191  * set or modify this value.
 192  */
 193 static unsigned int debug_ratelimit = 1;
 194
 195 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 196  * loading of a cpufreq driver, temporarily disabled when a new policy
 197  * is set, and disabled upon cpufreq driver removal
 198  */
 199 static unsigned int disable_ratelimit = 1;
 200 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 201
 202 static void cpufreq_debug_enable_ratelimit(void)
 203 {
 204         unsigned long flags;
 205
 206         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 207         if (disable_ratelimit)
 208                 disable_ratelimit--;
 209         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 210 }
 211
 212 static void cpufreq_debug_disable_ratelimit(void)
 213 {
 214         unsigned long flags;
 215
 216         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 217         disable_ratelimit++;
 218         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 219 }
 220
 221 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 222                         const char *fmt, ...)
 223 {
 224         char s[256];
 225         va_list args;
 226         unsigned int len;
 227         unsigned long flags;
 228
 229         WARN_ON(!prefix);
 230         if (type & debug) {
 231                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 232                 if (!disable_ratelimit && debug_ratelimit
 233                                         && !printk_ratelimit()) {
 234                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235                         return;
 236                 }
 237                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 238
 239                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 240
 241                 va_start(args, fmt);
 242                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 243                 va_end(args);
 244
 245                 printk(s);
 246
 247                 WARN_ON(len < 5);
 248         }
 249 }
 250 EXPORT_SYMBOL(cpufreq_debug_printk);
 251
 252
 253 module_param(debug, uint, 0644);
 254 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 255                         " 2 to debug drivers, and 4 to debug governors.");
 256
 257 module_param(debug_ratelimit, uint, 0644);
 258 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 259                                         " set to 0 to disable ratelimiting.");
 260
 261 #else /* !CONFIG_CPU_FREQ_DEBUG */
 262
 263 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 264 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 265
 266 #endif /* CONFIG_CPU_FREQ_DEBUG */
 267
 268
 269 /*********************************************************************
 270  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 271  *********************************************************************/
 272
 273 /**
 274  * adjust_jiffies - adjust the system "loops_per_jiffy"
 275  *
 276  * This function alters the system "loops_per_jiffy" for the clock
 277  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 278  * systems as each CPU might be scaled differently. So, use the arch
 279  * per-CPU loops_per_jiffy value wherever possible.
 280  */
 281 #ifndef CONFIG_SMP
 282 static unsigned long l_p_j_ref;
 283 static unsigned int  l_p_j_ref_freq;
 284
 285 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 286 {
 287         if (ci->flags & CPUFREQ_CONST_LOOPS)
 288                 return;
 289
 290         if (!l_p_j_ref_freq) {
 291                 l_p_j_ref = loops_per_jiffy;
 292                 l_p_j_ref_freq = ci->old;
 293                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 294                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 295         }
 296         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 297             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 298             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 299                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 300                                                                 ci->new);
 301                 dprintk("scaling loops_per_jiffy to %lu "
 302                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 303         }
 304 }
 305 #else
 306 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 307 {
 308         return;
 309 }
 310 #endif
 311
 312
 313 /**
 314  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 315  * on frequency transition.
 316  *
 317  * This function calls the transition notifiers and the "adjust_jiffies"
 318  * function. It is called twice on all CPU frequency changes that have
 319  * external effects.
 320  */
 321 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 322 {
 323         struct cpufreq_policy *policy;
 324
 325         BUG_ON(irqs_disabled());
 326
 327         freqs->flags = cpufreq_driver->flags;
 328         dprintk("notification %u of frequency transition to %u kHz\n",
 329                 state, freqs->new);
 330
 331         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 332         switch (state) {
 333
 334         case CPUFREQ_PRECHANGE:
 335                 /* detect if the driver reported a value as "old frequency"
 336                  * which is not equal to what the cpufreq core thinks is
 337                  * "old frequency".
 338                  */
 339                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 340                         if ((policy) && (policy->cpu == freqs->cpu) &&
 341                             (policy->cur) && (policy->cur != freqs->old)) {
 342                                 dprintk("Warning: CPU frequency is"
 343                                         " %u, cpufreq assumed %u kHz.\n",
 344                                         freqs->old, policy->cur);
 345                                 freqs->old = policy->cur;
 346                         }
 347                 }
 348                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 349                                 CPUFREQ_PRECHANGE, freqs);
 350                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 351                 break;
 352
 353         case CPUFREQ_POSTCHANGE:
 354                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 355                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 356                                 CPUFREQ_POSTCHANGE, freqs);
 357                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 358                         policy->cur = freqs->new;
 359                 break;
 360         }
 361 }
 362 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 363
 364
 365
 366 /*********************************************************************
 367  *                          SYSFS INTERFACE                          *
 368  *********************************************************************/
 369
 370 static struct cpufreq_governor *__find_governor(const char *str_governor)
 371 {
 372         struct cpufreq_governor *t;
 373
 374         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 375                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 376                         return t;
 377
 378         return NULL;
 379 }
 380
 381 /**
 382  * cpufreq_parse_governor - parse a governor string
 383  */
 384 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 385                                 struct cpufreq_governor **governor)
 386 {
 387         int err = -EINVAL;
 388
 389         if (!cpufreq_driver)
 390                 goto out;
 391
 392         if (cpufreq_driver->setpolicy) {
 393                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 394                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 395                         err = 0;
 396                 } else if (!strnicmp(str_governor, "powersave",
 397                                                 CPUFREQ_NAME_LEN)) {
 398                         *policy = CPUFREQ_POLICY_POWERSAVE;
 399                         err = 0;
 400                 }
 401         } else if (cpufreq_driver->target) {
 402                 struct cpufreq_governor *t;
 403
 404                 mutex_lock(&cpufreq_governor_mutex);
 405
 406                 t = __find_governor(str_governor);
 407
 408                 if (t == NULL) {
 409                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 410                                                                 str_governor);
 411
 412                         if (name) {
 413                                 int ret;
 414
 415                                 mutex_unlock(&cpufreq_governor_mutex);
 416                                 ret = request_module("%s", name);
 417                                 mutex_lock(&cpufreq_governor_mutex);
 418
 419                                 if (ret == 0)
 420                                         t = __find_governor(str_governor);
 421                         }
 422
 423                         kfree(name);
 424                 }
 425
 426                 if (t != NULL) {
 427                         *governor = t;
 428                         err = 0;
 429                 }
 430
 431                 mutex_unlock(&cpufreq_governor_mutex);
 432         }
 433 out:
 434         return err;
 435 }
 436
 437
 438 /**
 439  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 440  * print out cpufreq information
 441  *
 442  * Write out information from cpufreq_driver->policy[cpu]; object must be
 443  * "unsigned int".
 444  */
 445
 446 #define show_one(file_name, object)                     \
 447 static ssize_t show_##file_name                         \
 448 (struct cpufreq_policy *policy, char *buf)              \
 449 {                                                       \
 450         return sprintf(buf, "%u\n", policy->object);    \
 451 }
 452
 453 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 454 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 455 show_one(scaling_min_freq, min);
 456 show_one(scaling_max_freq, max);
 457 show_one(scaling_cur_freq, cur);
 458
 459 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 460                                 struct cpufreq_policy *policy);
 461
 462 /**
 463  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 464  */
 465 #define store_one(file_name, object)                    \
 466 static ssize_t store_##file_name                                        \
 467 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 468 {                                                                       \
 469         unsigned int ret = -EINVAL;                                     \
 470         struct cpufreq_policy new_policy;                               \
 471                                                                         \
 472         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 473         if (ret)                                                        \
 474                 return -EINVAL;                                         \
 475                                                                         \
 476         ret = sscanf(buf, "%u", &new_policy.object);                    \
 477         if (ret != 1)                                                   \
 478                 return -EINVAL;                                         \
 479                                                                         \
 480         ret = __cpufreq_set_policy(policy, &new_policy);                \
 481         policy->user_policy.object = policy->object;                    \
 482                                                                         \
 483         return ret ? ret : count;                                       \
 484 }
 485
 486 store_one(scaling_min_freq, min);
 487 store_one(scaling_max_freq, max);
 488
 489 /**
 490  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 491  */
 492 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 493                                         char *buf)
 494 {
 495         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 496         if (!cur_freq)
 497                 return sprintf(buf, "<unknown>");
 498         return sprintf(buf, "%u\n", cur_freq);
 499 }
 500
 501
 502 /**
 503  * show_scaling_governor - show the current policy for the specified CPU
 504  */
 505 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 506 {
 507         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 508                 return sprintf(buf, "powersave\n");
 509         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 510                 return sprintf(buf, "performance\n");
 511         else if (policy->governor)
 512                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 513                                 policy->governor->name);
 514         return -EINVAL;
 515 }
 516
 517
 518 /**
 519  * store_scaling_governor - store policy for the specified CPU
 520  */
 521 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 522                                         const char *buf, size_t count)
 523 {
 524         unsigned int ret = -EINVAL;
 525         char    str_governor[16];
 526         struct cpufreq_policy new_policy;
 527
 528         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 529         if (ret)
 530                 return ret;
 531
 532         ret = sscanf(buf, "%15s", str_governor);
 533         if (ret != 1)
 534                 return -EINVAL;
 535
 536         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 537                                                 &new_policy.governor))
 538                 return -EINVAL;
 539
 540         /* Do not use cpufreq_set_policy here or the user_policy.max
 541            will be wrongly overridden */
 542         ret = __cpufreq_set_policy(policy, &new_policy);
 543
 544         policy->user_policy.policy = policy->policy;
 545         policy->user_policy.governor = policy->governor;
 546
 547         if (ret)
 548                 return ret;
 549         else
 550                 return count;
 551 }
 552
 553 /**
 554  * show_scaling_driver - show the cpufreq driver currently loaded
 555  */
 556 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 557 {
 558         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 559 }
 560
 561 /**
 562  * show_scaling_available_governors - show the available CPUfreq governors
 563  */
 564 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 565                                                 char *buf)
 566 {
 567         ssize_t i = 0;
 568         struct cpufreq_governor *t;
 569
 570         if (!cpufreq_driver->target) {
 571                 i += sprintf(buf, "performance powersave");
 572                 goto out;
 573         }
 574
 575         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 576                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 577                     - (CPUFREQ_NAME_LEN + 2)))
 578                         goto out;
 579                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 580         }
 581 out:
 582         i += sprintf(&buf[i], "\n");
 583         return i;
 584 }
 585
 586 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 587 {
 588         ssize_t i = 0;
 589         unsigned int cpu;
 590
 591         for_each_cpu(cpu, mask) {
 592                 if (i)
 593                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 594                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 595                 if (i >= (PAGE_SIZE - 5))
 596                         break;
 597         }
 598         i += sprintf(&buf[i], "\n");
 599         return i;
 600 }
 601
 602 /**
 603  * show_related_cpus - show the CPUs affected by each transition even if
 604  * hw coordination is in use
 605  */
 606 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 607 {
 608         if (cpumask_empty(policy->related_cpus))
 609                 return show_cpus(policy->cpus, buf);
 610         return show_cpus(policy->related_cpus, buf);
 611 }
 612
 613 /**
 614  * show_affected_cpus - show the CPUs affected by each transition
 615  */
 616 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 617 {
 618         return show_cpus(policy->cpus, buf);
 619 }
 620
 621 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 622                                         const char *buf, size_t count)
 623 {
 624         unsigned int freq = 0;
 625         unsigned int ret;
 626
 627         if (!policy->governor || !policy->governor->store_setspeed)
 628                 return -EINVAL;
 629
 630         ret = sscanf(buf, "%u", &freq);
 631         if (ret != 1)
 632                 return -EINVAL;
 633
 634         policy->governor->store_setspeed(policy, freq);
 635
 636         return count;
 637 }
 638
 639 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 640 {
 641         if (!policy->governor || !policy->governor->show_setspeed)
 642                 return sprintf(buf, "<unsupported>\n");
 643
 644         return policy->governor->show_setspeed(policy, buf);
 645 }
 646
 647 #define define_one_ro(_name) \
 648 static struct freq_attr _name = \
 649 __ATTR(_name, 0444, show_##_name, NULL)
 650
 651 #define define_one_ro0400(_name) \
 652 static struct freq_attr _name = \
 653 __ATTR(_name, 0400, show_##_name, NULL)
 654
 655 #define define_one_rw(_name) \
 656 static struct freq_attr _name = \
 657 __ATTR(_name, 0644, show_##_name, store_##_name)
 658
 659 define_one_ro0400(cpuinfo_cur_freq);
 660 define_one_ro(cpuinfo_min_freq);
 661 define_one_ro(cpuinfo_max_freq);
 662 define_one_ro(scaling_available_governors);
 663 define_one_ro(scaling_driver);
 664 define_one_ro(scaling_cur_freq);
 665 define_one_ro(related_cpus);
 666 define_one_ro(affected_cpus);
 667 define_one_rw(scaling_min_freq);
 668 define_one_rw(scaling_max_freq);
 669 define_one_rw(scaling_governor);
 670 define_one_rw(scaling_setspeed);
 671
 672 static struct attribute *default_attrs[] = {
 673         &cpuinfo_min_freq.attr,
 674         &cpuinfo_max_freq.attr,
 675         &scaling_min_freq.attr,
 676         &scaling_max_freq.attr,
 677         &affected_cpus.attr,
 678         &related_cpus.attr,
 679         &scaling_governor.attr,
 680         &scaling_driver.attr,
 681         &scaling_available_governors.attr,
 682         &scaling_setspeed.attr,
 683         NULL
 684 };
 685
 686 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 687 #define to_attr(a) container_of(a, struct freq_attr, attr)
 688
 689 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 690 {
 691         struct cpufreq_policy *policy = to_policy(kobj);
 692         struct freq_attr *fattr = to_attr(attr);
 693         ssize_t ret = -EINVAL;
 694         policy = cpufreq_cpu_get(policy->cpu);
 695         if (!policy)
 696                 goto no_policy;
 697
 698         if (lock_policy_rwsem_read(policy->cpu) < 0)
 699                 goto fail;
 700
 701         if (fattr->show)
 702                 ret = fattr->show(policy, buf);
 703         else
 704                 ret = -EIO;
 705
 706         unlock_policy_rwsem_read(policy->cpu);
 707 fail:
 708         cpufreq_cpu_put(policy);
 709 no_policy:
 710         return ret;
 711 }
 712
 713 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 714                      const char *buf, size_t count)
 715 {
 716         struct cpufreq_policy *policy = to_policy(kobj);
 717         struct freq_attr *fattr = to_attr(attr);
 718         ssize_t ret = -EINVAL;
 719         policy = cpufreq_cpu_get(policy->cpu);
 720         if (!policy)
 721                 goto no_policy;
 722
 723         if (lock_policy_rwsem_write(policy->cpu) < 0)
 724                 goto fail;
 725
 726         if (fattr->store)
 727                 ret = fattr->store(policy, buf, count);
 728         else
 729                 ret = -EIO;
 730
 731         unlock_policy_rwsem_write(policy->cpu);
 732 fail:
 733         cpufreq_cpu_put(policy);
 734 no_policy:
 735         return ret;
 736 }
 737
 738 static void cpufreq_sysfs_release(struct kobject *kobj)
 739 {
 740         struct cpufreq_policy *policy = to_policy(kobj);
 741         dprintk("last reference is dropped\n");
 742         complete(&policy->kobj_unregister);
 743 }
 744
 745 static struct sysfs_ops sysfs_ops = {
 746         .show   = show,
 747         .store  = store,
 748 };
 749
 750 static struct kobj_type ktype_cpufreq = {
 751         .sysfs_ops      = &sysfs_ops,
 752         .default_attrs  = default_attrs,
 753         .release        = cpufreq_sysfs_release,
 754 };
 755
 756 static struct kobj_type ktype_empty_cpufreq = {
 757         .sysfs_ops      = &sysfs_ops,
 758         .release        = cpufreq_sysfs_release,
 759 };
 760
 761
 762 /**
 763  * cpufreq_add_dev - add a CPU device
 764  *
 765  * Adds the cpufreq interface for a CPU device.
 766  */
 767 static int cpufreq_add_dev(struct sys_device *sys_dev)
 768 {
 769         unsigned int cpu = sys_dev->id;
 770         int ret = 0;
 771         struct cpufreq_policy new_policy;
 772         struct cpufreq_policy *policy;
 773         struct freq_attr **drv_attr;
 774         struct sys_device *cpu_sys_dev;
 775         unsigned long flags;
 776         unsigned int j;
 777 #ifdef CONFIG_SMP
 778         struct cpufreq_policy *managed_policy;
 779 #endif
 780
 781         if (cpu_is_offline(cpu))
 782                 return 0;
 783
 784         cpufreq_debug_disable_ratelimit();
 785         dprintk("adding CPU %u\n", cpu);
 786
 787 #ifdef CONFIG_SMP
 788         /* check whether a different CPU already registered this
 789          * CPU because it is in the same boat. */
 790         policy = cpufreq_cpu_get(cpu);
 791         if (unlikely(policy)) {
 792                 cpufreq_cpu_put(policy);
 793                 cpufreq_debug_enable_ratelimit();
 794                 return 0;
 795         }
 796 #endif
 797
 798         if (!try_module_get(cpufreq_driver->owner)) {
 799                 ret = -EINVAL;
 800                 goto module_out;
 801         }
 802
 803         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 804         if (!policy) {
 805                 ret = -ENOMEM;
 806                 goto nomem_out;
 807         }
 808         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) {
 809                 kfree(policy);
 810                 ret = -ENOMEM;
 811                 goto nomem_out;
 812         }
 813         if (!alloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) {
 814                 free_cpumask_var(policy->cpus);
 815                 kfree(policy);
 816                 ret = -ENOMEM;
 817                 goto nomem_out;
 818         }
 819
 820         policy->cpu = cpu;
 821         cpumask_copy(policy->cpus, cpumask_of(cpu));
 822
 823         /* Initially set CPU itself as the policy_cpu */
 824         per_cpu(policy_cpu, cpu) = cpu;
 825         lock_policy_rwsem_write(cpu);
 826
 827         init_completion(&policy->kobj_unregister);
 828         INIT_WORK(&policy->update, handle_update);
 829
 830         /* Set governor before ->init, so that driver could check it */
 831         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 832         /* call driver. From then on the cpufreq must be able
 833          * to accept all calls to ->verify and ->setpolicy for this CPU
 834          */
 835         ret = cpufreq_driver->init(policy);
 836         if (ret) {
 837                 dprintk("initialization failed\n");
 838                 goto err_out;
 839         }
 840         policy->user_policy.min = policy->min;
 841         policy->user_policy.max = policy->max;
 842
 843         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 844                                      CPUFREQ_START, policy);
 845
 846 #ifdef CONFIG_SMP
 847
 848 #ifdef CONFIG_HOTPLUG_CPU
 849         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 850                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 851                 dprintk("Restoring governor %s for cpu %d\n",
 852                        policy->governor->name, cpu);
 853         }
 854 #endif
 855
 856         for_each_cpu(j, policy->cpus) {
 857                 if (cpu == j)
 858                         continue;
 859
 860                 /* Check for existing affected CPUs.
 861                  * They may not be aware of it due to CPU Hotplug.
 862                  */
 863                 managed_policy = cpufreq_cpu_get(j);            /* FIXME: Where is this released?  What about error paths? */
 864                 if (unlikely(managed_policy)) {
 865
 866                         /* Set proper policy_cpu */
 867                         unlock_policy_rwsem_write(cpu);
 868                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 869
 870                         if (lock_policy_rwsem_write(cpu) < 0)
 871                                 goto err_out_driver_exit;
 872
 873                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 874                         cpumask_copy(managed_policy->cpus, policy->cpus);
 875                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 876                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 877
 878                         dprintk("CPU already managed, adding link\n");
 879                         ret = sysfs_create_link(&sys_dev->kobj,
 880                                                 &managed_policy->kobj,
 881                                                 "cpufreq");
 882                         if (ret)
 883                                 goto err_out_driver_exit;
 884
 885                         cpufreq_debug_enable_ratelimit();
 886                         ret = 0;
 887                         goto err_out_driver_exit; /* call driver->exit() */
 888                 }
 889         }
 890 #endif
 891         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 892
 893         /* prepare interface data */
 894         if (!cpufreq_driver->hide_interface) {
 895                 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 896                                            &sys_dev->kobj, "cpufreq");
 897                 if (ret)
 898                         goto err_out_driver_exit;
 899
 900                 /* set up files for this cpu device */
 901                 drv_attr = cpufreq_driver->attr;
 902                 while ((drv_attr) && (*drv_attr)) {
 903                         ret = sysfs_create_file(&policy->kobj,
 904                                                 &((*drv_attr)->attr));
 905                         if (ret)
 906                                 goto err_out_driver_exit;
 907                         drv_attr++;
 908                 }
 909                 if (cpufreq_driver->get) {
 910                         ret = sysfs_create_file(&policy->kobj,
 911                                                 &cpuinfo_cur_freq.attr);
 912                         if (ret)
 913                                 goto err_out_driver_exit;
 914                 }
 915                 if (cpufreq_driver->target) {
 916                         ret = sysfs_create_file(&policy->kobj,
 917                                                 &scaling_cur_freq.attr);
 918                         if (ret)
 919                                 goto err_out_driver_exit;
 920                 }
 921         } else {
 922                 ret = kobject_init_and_add(&policy->kobj, &ktype_empty_cpufreq,
 923                                            &sys_dev->kobj, "cpufreq");
 924                 if (ret)
 925                         goto err_out_driver_exit;
 926         }
 927
 928         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 929         for_each_cpu(j, policy->cpus) {
 930                 per_cpu(cpufreq_cpu_data, j) = policy;
 931                 per_cpu(policy_cpu, j) = policy->cpu;
 932         }
 933         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 934
 935         /* symlink affected CPUs */
 936         for_each_cpu(j, policy->cpus) {
 937                 if (j == cpu)
 938                         continue;
 939                 if (!cpu_online(j))
 940                         continue;
 941
 942                 dprintk("CPU %u already managed, adding link\n", j);
 943                 cpufreq_cpu_get(cpu);
 944                 cpu_sys_dev = get_cpu_sysdev(j);
 945                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 946                                         "cpufreq");
 947                 if (ret)
 948                         goto err_out_unregister;
 949         }
 950
 951         policy->governor = NULL; /* to assure that the starting sequence is
 952                                   * run in cpufreq_set_policy */
 953
 954         /* set default policy */
 955         ret = __cpufreq_set_policy(policy, &new_policy);
 956         policy->user_policy.policy = policy->policy;
 957         policy->user_policy.governor = policy->governor;
 958
 959         if (ret) {
 960                 dprintk("setting policy failed\n");
 961                 goto err_out_unregister;
 962         }
 963
 964         unlock_policy_rwsem_write(cpu);
 965
 966         kobject_uevent(&policy->kobj, KOBJ_ADD);
 967         module_put(cpufreq_driver->owner);
 968         dprintk("initialization complete\n");
 969         cpufreq_debug_enable_ratelimit();
 970
 971         return 0;
 972
 973
 974 err_out_unregister:
 975         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 976         for_each_cpu(j, policy->cpus)
 977                 per_cpu(cpufreq_cpu_data, j) = NULL;
 978         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 979
 980         kobject_put(&policy->kobj);
 981         wait_for_completion(&policy->kobj_unregister);
 982
 983 err_out_driver_exit:
 984         if (cpufreq_driver->exit)
 985                 cpufreq_driver->exit(policy);
 986
 987 err_out:
 988         unlock_policy_rwsem_write(cpu);
 989         kfree(policy);
 990
 991 nomem_out:
 992         module_put(cpufreq_driver->owner);
 993 module_out:
 994         cpufreq_debug_enable_ratelimit();
 995         return ret;
 996 }
 997
 998
 999 /**
1000  * __cpufreq_remove_dev - remove a CPU device
1001  *
1002  * Removes the cpufreq interface for a CPU device.
1003  * Caller should already have policy_rwsem in write mode for this CPU.
1004  * This routine frees the rwsem before returning.
1005  */
1006 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1007 {
1008         unsigned int cpu = sys_dev->id;
1009         unsigned long flags;
1010         struct cpufreq_policy *data;
1011 #ifdef CONFIG_SMP
1012         struct sys_device *cpu_sys_dev;
1013         unsigned int j;
1014 #endif
1015
1016         cpufreq_debug_disable_ratelimit();
1017         dprintk("unregistering CPU %u\n", cpu);
1018
1019         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1020         data = per_cpu(cpufreq_cpu_data, cpu);
1021
1022         if (!data) {
1023                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1024                 cpufreq_debug_enable_ratelimit();
1025                 unlock_policy_rwsem_write(cpu);
1026                 return -EINVAL;
1027         }
1028         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1029
1030
1031 #ifdef CONFIG_SMP
1032         /* if this isn't the CPU which is the parent of the kobj, we
1033          * only need to unlink, put and exit
1034          */
1035         if (unlikely(cpu != data->cpu)) {
1036                 dprintk("removing link\n");
1037                 cpumask_clear_cpu(cpu, data->cpus);
1038                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1039                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1040                 cpufreq_cpu_put(data);
1041                 cpufreq_debug_enable_ratelimit();
1042                 unlock_policy_rwsem_write(cpu);
1043                 return 0;
1044         }
1045 #endif
1046
1047 #ifdef CONFIG_SMP
1048
1049 #ifdef CONFIG_HOTPLUG_CPU
1050         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1051 #endif
1052
1053         /* if we have other CPUs still registered, we need to unlink them,
1054          * or else wait_for_completion below will lock up. Clean the
1055          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1056          * the sysfs links afterwards.
1057          */
1058         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1059                 for_each_cpu(j, data->cpus) {
1060                         if (j == cpu)
1061                                 continue;
1062                         per_cpu(cpufreq_cpu_data, j) = NULL;
1063                 }
1064         }
1065
1066         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1067
1068         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1069                 for_each_cpu(j, data->cpus) {
1070                         if (j == cpu)
1071                                 continue;
1072                         dprintk("removing link for cpu %u\n", j);
1073 #ifdef CONFIG_HOTPLUG_CPU
1074                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1075 #endif
1076                         cpu_sys_dev = get_cpu_sysdev(j);
1077                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1078                         cpufreq_cpu_put(data);
1079                 }
1080         }
1081 #else
1082         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1083 #endif
1084
1085         if (cpufreq_driver->target)
1086                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1087
1088         unlock_policy_rwsem_write(cpu);
1089
1090         kobject_put(&data->kobj);
1091
1092         /* we need to make sure that the underlying kobj is actually
1093          * not referenced anymore by anybody before we proceed with
1094          * unloading.
1095          */
1096         dprintk("waiting for dropping of refcount\n");
1097         wait_for_completion(&data->kobj_unregister);
1098         dprintk("wait complete\n");
1099
1100         if (cpufreq_driver->exit)
1101                 cpufreq_driver->exit(data);
1102
1103         free_cpumask_var(data->related_cpus);
1104         free_cpumask_var(data->cpus);
1105         kfree(data);
1106         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1107
1108         cpufreq_debug_enable_ratelimit();
1109         return 0;
1110 }
1111
1112
1113 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1114 {
1115         unsigned int cpu = sys_dev->id;
1116         int retval;
1117
1118         if (cpu_is_offline(cpu))
1119                 return 0;
1120
1121         if (unlikely(lock_policy_rwsem_write(cpu)))
1122                 BUG();
1123
1124         retval = __cpufreq_remove_dev(sys_dev);
1125         return retval;
1126 }
1127
1128
1129 static void handle_update(struct work_struct *work)
1130 {
1131         struct cpufreq_policy *policy =
1132                 container_of(work, struct cpufreq_policy, update);
1133         unsigned int cpu = policy->cpu;
1134         dprintk("handle_update for cpu %u called\n", cpu);
1135         cpufreq_update_policy(cpu);
1136 }
1137
1138 /**
1139  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1140  *      @cpu: cpu number
1141  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1142  *      @new_freq: CPU frequency the CPU actually runs at
1143  *
1144  *      We adjust to current frequency first, and need to clean up later.
1145  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1146  */
1147 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1148                                 unsigned int new_freq)
1149 {
1150         struct cpufreq_freqs freqs;
1151
1152         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1153                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1154
1155         freqs.cpu = cpu;
1156         freqs.old = old_freq;
1157         freqs.new = new_freq;
1158         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1159         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1160 }
1161
1162
1163 /**
1164  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1165  * @cpu: CPU number
1166  *
1167  * This is the last known freq, without actually getting it from the driver.
1168  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1169  */
1170 unsigned int cpufreq_quick_get(unsigned int cpu)
1171 {
1172         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1173         unsigned int ret_freq = 0;
1174
1175         if (policy) {
1176                 ret_freq = policy->cur;
1177                 cpufreq_cpu_put(policy);
1178         }
1179
1180         return ret_freq;
1181 }
1182 EXPORT_SYMBOL(cpufreq_quick_get);
1183
1184
1185 static unsigned int __cpufreq_get(unsigned int cpu)
1186 {
1187         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1188         unsigned int ret_freq = 0;
1189
1190         if (!cpufreq_driver->get)
1191                 return ret_freq;
1192
1193         ret_freq = cpufreq_driver->get(cpu);
1194
1195         if (ret_freq && policy->cur &&
1196                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1197                 /* verify no discrepancy between actual and
1198                                         saved value exists */
1199                 if (unlikely(ret_freq != policy->cur)) {
1200                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1201                         schedule_work(&policy->update);
1202                 }
1203         }
1204
1205         return ret_freq;
1206 }
1207
1208 /**
1209  * cpufreq_get - get the current CPU frequency (in kHz)
1210  * @cpu: CPU number
1211  *
1212  * Get the CPU current (static) CPU frequency
1213  */
1214 unsigned int cpufreq_get(unsigned int cpu)
1215 {
1216         unsigned int ret_freq = 0;
1217         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1218
1219         if (!policy)
1220                 goto out;
1221
1222         if (unlikely(lock_policy_rwsem_read(cpu)))
1223                 goto out_policy;
1224
1225         ret_freq = __cpufreq_get(cpu);
1226
1227         unlock_policy_rwsem_read(cpu);
1228
1229 out_policy:
1230         cpufreq_cpu_put(policy);
1231 out:
1232         return ret_freq;
1233 }
1234 EXPORT_SYMBOL(cpufreq_get);
1235
1236
1237 /**
1238  *      cpufreq_suspend - let the low level driver prepare for suspend
1239  */
1240
1241 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1242 {
1243         int cpu = sysdev->id;
1244         int ret = 0;
1245         unsigned int cur_freq = 0;
1246         struct cpufreq_policy *cpu_policy;
1247
1248         dprintk("suspending cpu %u\n", cpu);
1249
1250         if (!cpu_online(cpu))
1251                 return 0;
1252
1253         /* we may be lax here as interrupts are off. Nonetheless
1254          * we need to grab the correct cpu policy, as to check
1255          * whether we really run on this CPU.
1256          */
1257
1258         cpu_policy = cpufreq_cpu_get(cpu);
1259         if (!cpu_policy)
1260                 return -EINVAL;
1261
1262         /* only handle each CPU group once */
1263         if (unlikely(cpu_policy->cpu != cpu))
1264                 goto out;
1265
1266         if (cpufreq_driver->suspend) {
1267                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1268                 if (ret) {
1269                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1270                                         "step on CPU %u\n", cpu_policy->cpu);
1271                         goto out;
1272                 }
1273         }
1274
1275         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1276                 goto out;
1277
1278         if (cpufreq_driver->get)
1279                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1280
1281         if (!cur_freq || !cpu_policy->cur) {
1282                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1283                        "frequency is what timing core thinks it is.\n");
1284                 goto out;
1285         }
1286
1287         if (unlikely(cur_freq != cpu_policy->cur)) {
1288                 struct cpufreq_freqs freqs;
1289
1290                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1291                         dprintk("Warning: CPU frequency is %u, "
1292                                "cpufreq assumed %u kHz.\n",
1293                                cur_freq, cpu_policy->cur);
1294
1295                 freqs.cpu = cpu;
1296                 freqs.old = cpu_policy->cur;
1297                 freqs.new = cur_freq;
1298
1299                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1300                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1301                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1302
1303                 cpu_policy->cur = cur_freq;
1304         }
1305
1306 out:
1307         cpufreq_cpu_put(cpu_policy);
1308         return ret;
1309 }
1310
1311 /**
1312  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1313  *
1314  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1315  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1316  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1317  *          restored.
1318  */
1319 static int cpufreq_resume(struct sys_device *sysdev)
1320 {
1321         int cpu = sysdev->id;
1322         int ret = 0;
1323         struct cpufreq_policy *cpu_policy;
1324
1325         dprintk("resuming cpu %u\n", cpu);
1326
1327         if (!cpu_online(cpu))
1328                 return 0;
1329
1330         /* we may be lax here as interrupts are off. Nonetheless
1331          * we need to grab the correct cpu policy, as to check
1332          * whether we really run on this CPU.
1333          */
1334
1335         cpu_policy = cpufreq_cpu_get(cpu);
1336         if (!cpu_policy)
1337                 return -EINVAL;
1338
1339         /* only handle each CPU group once */
1340         if (unlikely(cpu_policy->cpu != cpu))
1341                 goto fail;
1342
1343         if (cpufreq_driver->resume) {
1344                 ret = cpufreq_driver->resume(cpu_policy);
1345                 if (ret) {
1346                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1347                                         "step on CPU %u\n", cpu_policy->cpu);
1348                         goto fail;
1349                 }
1350         }
1351
1352         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1353                 unsigned int cur_freq = 0;
1354
1355                 if (cpufreq_driver->get)
1356                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1357
1358                 if (!cur_freq || !cpu_policy->cur) {
1359                         printk(KERN_ERR "cpufreq: resume failed to assert "
1360                                         "current frequency is what timing core "
1361                                         "thinks it is.\n");
1362                         goto out;
1363                 }
1364
1365                 if (unlikely(cur_freq != cpu_policy->cur)) {
1366                         struct cpufreq_freqs freqs;
1367
1368                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1369                                 dprintk("Warning: CPU frequency "
1370                                        "is %u, cpufreq assumed %u kHz.\n",
1371                                        cur_freq, cpu_policy->cur);
1372
1373                         freqs.cpu = cpu;
1374                         freqs.old = cpu_policy->cur;
1375                         freqs.new = cur_freq;
1376
1377                         srcu_notifier_call_chain(
1378                                         &cpufreq_transition_notifier_list,
1379                                         CPUFREQ_RESUMECHANGE, &freqs);
1380                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1381
1382                         cpu_policy->cur = cur_freq;
1383                 }
1384         }
1385
1386 out:
1387         schedule_work(&cpu_policy->update);
1388 fail:
1389         cpufreq_cpu_put(cpu_policy);
1390         return ret;
1391 }
1392
1393 static struct sysdev_driver cpufreq_sysdev_driver = {
1394         .add            = cpufreq_add_dev,
1395         .remove         = cpufreq_remove_dev,
1396         .suspend        = cpufreq_suspend,
1397         .resume         = cpufreq_resume,
1398 };
1399
1400
1401 /*********************************************************************
1402  *                     NOTIFIER LISTS INTERFACE                      *
1403  *********************************************************************/
1404
1405 /**
1406  *      cpufreq_register_notifier - register a driver with cpufreq
1407  *      @nb: notifier function to register
1408  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1409  *
1410  *      Add a driver to one of two lists: either a list of drivers that
1411  *      are notified about clock rate changes (once before and once after
1412  *      the transition), or a list of drivers that are notified about
1413  *      changes in cpufreq policy.
1414  *
1415  *      This function may sleep, and has the same return conditions as
1416  *      blocking_notifier_chain_register.
1417  */
1418 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1419 {
1420         int ret;
1421
1422         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1423
1424         switch (list) {
1425         case CPUFREQ_TRANSITION_NOTIFIER:
1426                 ret = srcu_notifier_chain_register(
1427                                 &cpufreq_transition_notifier_list, nb);
1428                 break;
1429         case CPUFREQ_POLICY_NOTIFIER:
1430                 ret = blocking_notifier_chain_register(
1431                                 &cpufreq_policy_notifier_list, nb);
1432                 break;
1433         default:
1434                 ret = -EINVAL;
1435         }
1436
1437         return ret;
1438 }
1439 EXPORT_SYMBOL(cpufreq_register_notifier);
1440
1441
1442 /**
1443  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1444  *      @nb: notifier block to be unregistered
1445  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1446  *
1447  *      Remove a driver from the CPU frequency notifier list.
1448  *
1449  *      This function may sleep, and has the same return conditions as
1450  *      blocking_notifier_chain_unregister.
1451  */
1452 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1453 {
1454         int ret;
1455
1456         switch (list) {
1457         case CPUFREQ_TRANSITION_NOTIFIER:
1458                 ret = srcu_notifier_chain_unregister(
1459                                 &cpufreq_transition_notifier_list, nb);
1460                 break;
1461         case CPUFREQ_POLICY_NOTIFIER:
1462                 ret = blocking_notifier_chain_unregister(
1463                                 &cpufreq_policy_notifier_list, nb);
1464                 break;
1465         default:
1466                 ret = -EINVAL;
1467         }
1468
1469         return ret;
1470 }
1471 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1472
1473
1474 /*********************************************************************
1475  *                              GOVERNORS                            *
1476  *********************************************************************/
1477
1478
1479 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1480                             unsigned int target_freq,
1481                             unsigned int relation)
1482 {
1483         int retval = -EINVAL;
1484
1485         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1486                 target_freq, relation);
1487         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1488                 retval = cpufreq_driver->target(policy, target_freq, relation);
1489
1490         return retval;
1491 }
1492 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1493
1494 int cpufreq_driver_target(struct cpufreq_policy *policy,
1495                           unsigned int target_freq,
1496                           unsigned int relation)
1497 {
1498         int ret = -EINVAL;
1499
1500         policy = cpufreq_cpu_get(policy->cpu);
1501         if (!policy)
1502                 goto no_policy;
1503
1504         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1505                 goto fail;
1506
1507         ret = __cpufreq_driver_target(policy, target_freq, relation);
1508
1509         unlock_policy_rwsem_write(policy->cpu);
1510
1511 fail:
1512         cpufreq_cpu_put(policy);
1513 no_policy:
1514         return ret;
1515 }
1516 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1517
1518 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1519 {
1520         int ret = 0;
1521
1522         policy = cpufreq_cpu_get(policy->cpu);
1523         if (!policy)
1524                 return -EINVAL;
1525
1526         if (cpu_online(cpu) && cpufreq_driver->getavg)
1527                 ret = cpufreq_driver->getavg(policy, cpu);
1528
1529         cpufreq_cpu_put(policy);
1530         return ret;
1531 }
1532 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1533
1534 /*
1535  * when "event" is CPUFREQ_GOV_LIMITS
1536  */
1537
1538 static int __cpufreq_governor(struct cpufreq_policy *policy,
1539                                         unsigned int event)
1540 {
1541         int ret;
1542
1543         /* Only must be defined when default governor is known to have latency
1544            restrictions, like e.g. conservative or ondemand.
1545            That this is the case is already ensured in Kconfig
1546         */
1547 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1548         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1549 #else
1550         struct cpufreq_governor *gov = NULL;
1551 #endif
1552
1553         if (policy->governor->max_transition_latency &&
1554             policy->cpuinfo.transition_latency >
1555             policy->governor->max_transition_latency) {
1556                 if (!gov)
1557                         return -EINVAL;
1558                 else {
1559                         printk(KERN_WARNING "%s governor failed, too long"
1560                                " transition latency of HW, fallback"
1561                                " to %s governor\n",
1562                                policy->governor->name,
1563                                gov->name);
1564                         policy->governor = gov;
1565                 }
1566         }
1567
1568         if (!try_module_get(policy->governor->owner))
1569                 return -EINVAL;
1570
1571         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1572                                                 policy->cpu, event);
1573         ret = policy->governor->governor(policy, event);
1574
1575         /* we keep one module reference alive for
1576                         each CPU governed by this CPU */
1577         if ((event != CPUFREQ_GOV_START) || ret)
1578                 module_put(policy->governor->owner);
1579         if ((event == CPUFREQ_GOV_STOP) && !ret)
1580                 module_put(policy->governor->owner);
1581
1582         return ret;
1583 }
1584
1585
1586 int cpufreq_register_governor(struct cpufreq_governor *governor)
1587 {
1588         int err;
1589
1590         if (!governor)
1591                 return -EINVAL;
1592
1593         mutex_lock(&cpufreq_governor_mutex);
1594
1595         err = -EBUSY;
1596         if (__find_governor(governor->name) == NULL) {
1597                 err = 0;
1598                 list_add(&governor->governor_list, &cpufreq_governor_list);
1599         }
1600
1601         mutex_unlock(&cpufreq_governor_mutex);
1602         return err;
1603 }
1604 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1605
1606
1607 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1608 {
1609         if (!governor)
1610                 return;
1611
1612         mutex_lock(&cpufreq_governor_mutex);
1613         list_del(&governor->governor_list);
1614         mutex_unlock(&cpufreq_governor_mutex);
1615         return;
1616 }
1617 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1618
1619
1620
1621 /*********************************************************************
1622  *                          POLICY INTERFACE                         *
1623  *********************************************************************/
1624
1625 /**
1626  * cpufreq_get_policy - get the current cpufreq_policy
1627  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1628  *      is written
1629  *
1630  * Reads the current cpufreq policy.
1631  */
1632 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1633 {
1634         struct cpufreq_policy *cpu_policy;
1635         if (!policy)
1636                 return -EINVAL;
1637
1638         cpu_policy = cpufreq_cpu_get(cpu);
1639         if (!cpu_policy)
1640                 return -EINVAL;
1641
1642         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1643
1644         cpufreq_cpu_put(cpu_policy);
1645         return 0;
1646 }
1647 EXPORT_SYMBOL(cpufreq_get_policy);
1648
1649
1650 /*
1651  * data   : current policy.
1652  * policy : policy to be set.
1653  */
1654 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1655                                 struct cpufreq_policy *policy)
1656 {
1657         int ret = 0;
1658
1659         cpufreq_debug_disable_ratelimit();
1660         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1661                 policy->min, policy->max);
1662
1663         memcpy(&policy->cpuinfo, &data->cpuinfo,
1664                                 sizeof(struct cpufreq_cpuinfo));
1665
1666         if (policy->min > data->max || policy->max < data->min) {
1667                 ret = -EINVAL;
1668                 goto error_out;
1669         }
1670
1671         /* verify the cpu speed can be set within this limit */
1672         ret = cpufreq_driver->verify(policy);
1673         if (ret)
1674                 goto error_out;
1675
1676         /* adjust if necessary - all reasons */
1677         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1678                         CPUFREQ_ADJUST, policy);
1679
1680         /* adjust if necessary - hardware incompatibility*/
1681         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1682                         CPUFREQ_INCOMPATIBLE, policy);
1683
1684         /* verify the cpu speed can be set within this limit,
1685            which might be different to the first one */
1686         ret = cpufreq_driver->verify(policy);
1687         if (ret)
1688                 goto error_out;
1689
1690         /* notification of the new policy */
1691         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1692                         CPUFREQ_NOTIFY, policy);
1693
1694         data->min = policy->min;
1695         data->max = policy->max;
1696
1697         dprintk("new min and max freqs are %u - %u kHz\n",
1698                                         data->min, data->max);
1699
1700         if (cpufreq_driver->setpolicy) {
1701                 data->policy = policy->policy;
1702                 dprintk("setting range\n");
1703                 ret = cpufreq_driver->setpolicy(policy);
1704         } else {
1705                 if (policy->governor != data->governor) {
1706                         /* save old, working values */
1707                         struct cpufreq_governor *old_gov = data->governor;
1708
1709                         dprintk("governor switch\n");
1710
1711                         /* end old governor */
1712                         if (data->governor)
1713                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1714
1715                         /* start new governor */
1716                         data->governor = policy->governor;
1717                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1718                                 /* new governor failed, so re-start old one */
1719                                 dprintk("starting governor %s failed\n",
1720                                                         data->governor->name);
1721                                 if (old_gov) {
1722                                         data->governor = old_gov;
1723                                         __cpufreq_governor(data,
1724                                                            CPUFREQ_GOV_START);
1725                                 }
1726                                 ret = -EINVAL;
1727                                 goto error_out;
1728                         }
1729                         /* might be a policy change, too, so fall through */
1730                 }
1731                 dprintk("governor: change or update limits\n");
1732                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1733         }
1734
1735 error_out:
1736         cpufreq_debug_enable_ratelimit();
1737         return ret;
1738 }
1739
1740 /**
1741  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1742  *      @cpu: CPU which shall be re-evaluated
1743  *
1744  *      Usefull for policy notifiers which have different necessities
1745  *      at different times.
1746  */
1747 int cpufreq_update_policy(unsigned int cpu)
1748 {
1749         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1750         struct cpufreq_policy policy;
1751         int ret;
1752
1753         if (!data) {
1754                 ret = -ENODEV;
1755                 goto no_policy;
1756         }
1757
1758         if (unlikely(lock_policy_rwsem_write(cpu))) {
1759                 ret = -EINVAL;
1760                 goto fail;
1761         }
1762
1763         dprintk("updating policy for CPU %u\n", cpu);
1764         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1765         policy.min = data->user_policy.min;
1766         policy.max = data->user_policy.max;
1767         policy.policy = data->user_policy.policy;
1768         policy.governor = data->user_policy.governor;
1769
1770         /* BIOS might change freq behind our back
1771           -> ask driver for current freq and notify governors about a change */
1772         if (cpufreq_driver->get) {
1773                 policy.cur = cpufreq_driver->get(cpu);
1774                 if (!data->cur) {
1775                         dprintk("Driver did not initialize current freq");
1776                         data->cur = policy.cur;
1777                 } else {
1778                         if (data->cur != policy.cur)
1779                                 cpufreq_out_of_sync(cpu, data->cur,
1780                                                                 policy.cur);
1781                 }
1782         }
1783
1784         ret = __cpufreq_set_policy(data, &policy);
1785
1786         unlock_policy_rwsem_write(cpu);
1787
1788 fail:
1789         cpufreq_cpu_put(data);
1790 no_policy:
1791         return ret;
1792 }
1793 EXPORT_SYMBOL(cpufreq_update_policy);
1794
1795 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1796                                         unsigned long action, void *hcpu)
1797 {
1798         unsigned int cpu = (unsigned long)hcpu;
1799         struct sys_device *sys_dev;
1800
1801         sys_dev = get_cpu_sysdev(cpu);
1802         if (sys_dev) {
1803                 switch (action) {
1804                 case CPU_ONLINE:
1805                 case CPU_ONLINE_FROZEN:
1806                         cpufreq_add_dev(sys_dev);
1807                         break;
1808                 case CPU_DOWN_PREPARE:
1809                 case CPU_DOWN_PREPARE_FROZEN:
1810                         if (unlikely(lock_policy_rwsem_write(cpu)))
1811                                 BUG();
1812
1813                         __cpufreq_remove_dev(sys_dev);
1814                         break;
1815                 case CPU_DOWN_FAILED:
1816                 case CPU_DOWN_FAILED_FROZEN:
1817                         cpufreq_add_dev(sys_dev);
1818                         break;
1819                 }
1820         }
1821         return NOTIFY_OK;
1822 }
1823
1824 static struct notifier_block __refdata cpufreq_cpu_notifier =
1825 {
1826     .notifier_call = cpufreq_cpu_callback,
1827 };
1828
1829 /*********************************************************************
1830  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1831  *********************************************************************/
1832
1833 /**
1834  * cpufreq_register_driver - register a CPU Frequency driver
1835  * @driver_data: A struct cpufreq_driver containing the values#
1836  * submitted by the CPU Frequency driver.
1837  *
1838  *   Registers a CPU Frequency driver to this core code. This code
1839  * returns zero on success, -EBUSY when another driver got here first
1840  * (and isn't unregistered in the meantime).
1841  *
1842  */
1843 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1844 {
1845         unsigned long flags;
1846         int ret;
1847
1848         if (!driver_data || !driver_data->verify || !driver_data->init ||
1849             ((!driver_data->setpolicy) && (!driver_data->target)))
1850                 return -EINVAL;
1851
1852         dprintk("trying to register driver %s\n", driver_data->name);
1853
1854         if (driver_data->setpolicy)
1855                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1856
1857         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1858         if (cpufreq_driver) {
1859                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1860                 return -EBUSY;
1861         }
1862         cpufreq_driver = driver_data;
1863         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1864
1865         ret = sysdev_driver_register(&cpu_sysdev_class,
1866                                         &cpufreq_sysdev_driver);
1867
1868         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1869                 int i;
1870                 ret = -ENODEV;
1871
1872                 /* check for at least one working CPU */
1873                 for (i = 0; i < nr_cpu_ids; i++)
1874                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1875                                 ret = 0;
1876                                 break;
1877                         }
1878
1879                 /* if all ->init() calls failed, unregister */
1880                 if (ret) {
1881                         dprintk("no CPU initialized for driver %s\n",
1882                                                         driver_data->name);
1883                         sysdev_driver_unregister(&cpu_sysdev_class,
1884                                                 &cpufreq_sysdev_driver);
1885
1886                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1887                         cpufreq_driver = NULL;
1888                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1889                 }
1890         }
1891
1892         if (!ret) {
1893                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1894                 dprintk("driver %s up and running\n", driver_data->name);
1895                 cpufreq_debug_enable_ratelimit();
1896         }
1897
1898         return ret;
1899 }
1900 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1901
1902
1903 /**
1904  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1905  *
1906  *    Unregister the current CPUFreq driver. Only call this if you have
1907  * the right to do so, i.e. if you have succeeded in initialising before!
1908  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1909  * currently not initialised.
1910  */
1911 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1912 {
1913         unsigned long flags;
1914
1915         cpufreq_debug_disable_ratelimit();
1916
1917         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1918                 cpufreq_debug_enable_ratelimit();
1919                 return -EINVAL;
1920         }
1921
1922         dprintk("unregistering driver %s\n", driver->name);
1923
1924         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1925         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1926
1927         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1928         cpufreq_driver = NULL;
1929         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1930
1931         return 0;
1932 }
1933 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1934
1935 static int __init cpufreq_core_init(void)
1936 {
1937         int cpu;
1938
1939         for_each_possible_cpu(cpu) {
1940                 per_cpu(policy_cpu, cpu) = -1;
1941                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1942         }
1943         return 0;
1944 }
1945
1946 core_initcall(cpufreq_core_init);