drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static struct cpufreq_policy *cpufreq_cpu_data[NR_CPUS];
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static struct cpufreq_governor *cpufreq_cpu_governor[NR_CPUS];
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static int __init init_cpufreq_transition_notifier_list(void)
 122 {
 123         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 124         return 0;
 125 }
 126 pure_initcall(init_cpufreq_transition_notifier_list);
 127
 128 static LIST_HEAD(cpufreq_governor_list);
 129 static DEFINE_MUTEX (cpufreq_governor_mutex);
 130
 131 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 132 {
 133         struct cpufreq_policy *data;
 134         unsigned long flags;
 135
 136         if (cpu >= NR_CPUS)
 137                 goto err_out;
 138
 139         /* get the cpufreq driver */
 140         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 141
 142         if (!cpufreq_driver)
 143                 goto err_out_unlock;
 144
 145         if (!try_module_get(cpufreq_driver->owner))
 146                 goto err_out_unlock;
 147
 148
 149         /* get the CPU */
 150         data = cpufreq_cpu_data[cpu];
 151
 152         if (!data)
 153                 goto err_out_put_module;
 154
 155         if (!kobject_get(&data->kobj))
 156                 goto err_out_put_module;
 157
 158         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 159         return data;
 160
 161 err_out_put_module:
 162         module_put(cpufreq_driver->owner);
 163 err_out_unlock:
 164         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 165 err_out:
 166         return NULL;
 167 }
 168 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 169
 170
 171 void cpufreq_cpu_put(struct cpufreq_policy *data)
 172 {
 173         kobject_put(&data->kobj);
 174         module_put(cpufreq_driver->owner);
 175 }
 176 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 177
 178
 179 /*********************************************************************
 180  *                     UNIFIED DEBUG HELPERS                         *
 181  *********************************************************************/
 182 #ifdef CONFIG_CPU_FREQ_DEBUG
 183
 184 /* what part(s) of the CPUfreq subsystem are debugged? */
 185 static unsigned int debug;
 186
 187 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 188  * set or modify this value.
 189  */
 190 static unsigned int debug_ratelimit = 1;
 191
 192 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 193  * loading of a cpufreq driver, temporarily disabled when a new policy
 194  * is set, and disabled upon cpufreq driver removal
 195  */
 196 static unsigned int disable_ratelimit = 1;
 197 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 198
 199 static void cpufreq_debug_enable_ratelimit(void)
 200 {
 201         unsigned long flags;
 202
 203         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 204         if (disable_ratelimit)
 205                 disable_ratelimit--;
 206         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 207 }
 208
 209 static void cpufreq_debug_disable_ratelimit(void)
 210 {
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 214         disable_ratelimit++;
 215         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 216 }
 217
 218 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 219                                                         const char *fmt, ...)
 220 {
 221         char s[256];
 222         va_list args;
 223         unsigned int len;
 224         unsigned long flags;
 225
 226         WARN_ON(!prefix);
 227         if (type & debug) {
 228                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 229                 if (!disable_ratelimit && debug_ratelimit
 230                                         && !printk_ratelimit()) {
 231                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 232                         return;
 233                 }
 234                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235
 236                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 237
 238                 va_start(args, fmt);
 239                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 240                 va_end(args);
 241
 242                 printk(s);
 243
 244                 WARN_ON(len < 5);
 245         }
 246 }
 247 EXPORT_SYMBOL(cpufreq_debug_printk);
 248
 249
 250 module_param(debug, uint, 0644);
 251 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 252                         " 2 to debug drivers, and 4 to debug governors.");
 253
 254 module_param(debug_ratelimit, uint, 0644);
 255 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 256                                         " set to 0 to disable ratelimiting.");
 257
 258 #else /* !CONFIG_CPU_FREQ_DEBUG */
 259
 260 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 261 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 262
 263 #endif /* CONFIG_CPU_FREQ_DEBUG */
 264
 265
 266 /*********************************************************************
 267  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 268  *********************************************************************/
 269
 270 /**
 271  * adjust_jiffies - adjust the system "loops_per_jiffy"
 272  *
 273  * This function alters the system "loops_per_jiffy" for the clock
 274  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 275  * systems as each CPU might be scaled differently. So, use the arch
 276  * per-CPU loops_per_jiffy value wherever possible.
 277  */
 278 #ifndef CONFIG_SMP
 279 static unsigned long l_p_j_ref;
 280 static unsigned int  l_p_j_ref_freq;
 281
 282 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 283 {
 284         if (ci->flags & CPUFREQ_CONST_LOOPS)
 285                 return;
 286
 287         if (!l_p_j_ref_freq) {
 288                 l_p_j_ref = loops_per_jiffy;
 289                 l_p_j_ref_freq = ci->old;
 290                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 291                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 292         }
 293         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 294             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 295             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 296                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 297                                                                 ci->new);
 298                 dprintk("scaling loops_per_jiffy to %lu "
 299                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 300         }
 301 }
 302 #else
 303 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 304 {
 305         return;
 306 }
 307 #endif
 308
 309
 310 /**
 311  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 312  * on frequency transition.
 313  *
 314  * This function calls the transition notifiers and the "adjust_jiffies"
 315  * function. It is called twice on all CPU frequency changes that have
 316  * external effects.
 317  */
 318 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 319 {
 320         struct cpufreq_policy *policy;
 321
 322         BUG_ON(irqs_disabled());
 323
 324         freqs->flags = cpufreq_driver->flags;
 325         dprintk("notification %u of frequency transition to %u kHz\n",
 326                 state, freqs->new);
 327
 328         policy = cpufreq_cpu_data[freqs->cpu];
 329         switch (state) {
 330
 331         case CPUFREQ_PRECHANGE:
 332                 /* detect if the driver reported a value as "old frequency"
 333                  * which is not equal to what the cpufreq core thinks is
 334                  * "old frequency".
 335                  */
 336                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 337                         if ((policy) && (policy->cpu == freqs->cpu) &&
 338                             (policy->cur) && (policy->cur != freqs->old)) {
 339                                 dprintk("Warning: CPU frequency is"
 340                                         " %u, cpufreq assumed %u kHz.\n",
 341                                         freqs->old, policy->cur);
 342                                 freqs->old = policy->cur;
 343                         }
 344                 }
 345                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 346                                 CPUFREQ_PRECHANGE, freqs);
 347                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 348                 break;
 349
 350         case CPUFREQ_POSTCHANGE:
 351                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 352                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 353                                 CPUFREQ_POSTCHANGE, freqs);
 354                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 355                         policy->cur = freqs->new;
 356                 break;
 357         }
 358 }
 359 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 360
 361
 362
 363 /*********************************************************************
 364  *                          SYSFS INTERFACE                          *
 365  *********************************************************************/
 366
 367 static struct cpufreq_governor *__find_governor(const char *str_governor)
 368 {
 369         struct cpufreq_governor *t;
 370
 371         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 372                 if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 373                         return t;
 374
 375         return NULL;
 376 }
 377
 378 /**
 379  * cpufreq_parse_governor - parse a governor string
 380  */
 381 static int cpufreq_parse_governor (char *str_governor, unsigned int *policy,
 382                                 struct cpufreq_governor **governor)
 383 {
 384         int err = -EINVAL;
 385
 386         if (!cpufreq_driver)
 387                 goto out;
 388
 389         if (cpufreq_driver->setpolicy) {
 390                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 391                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 392                         err = 0;
 393                 } else if (!strnicmp(str_governor, "powersave",
 394                                                 CPUFREQ_NAME_LEN)) {
 395                         *policy = CPUFREQ_POLICY_POWERSAVE;
 396                         err = 0;
 397                 }
 398         } else if (cpufreq_driver->target) {
 399                 struct cpufreq_governor *t;
 400
 401                 mutex_lock(&cpufreq_governor_mutex);
 402
 403                 t = __find_governor(str_governor);
 404
 405                 if (t == NULL) {
 406                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 407                                                                 str_governor);
 408
 409                         if (name) {
 410                                 int ret;
 411
 412                                 mutex_unlock(&cpufreq_governor_mutex);
 413                                 ret = request_module(name);
 414                                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                                 if (ret == 0)
 417                                         t = __find_governor(str_governor);
 418                         }
 419
 420                         kfree(name);
 421                 }
 422
 423                 if (t != NULL) {
 424                         *governor = t;
 425                         err = 0;
 426                 }
 427
 428                 mutex_unlock(&cpufreq_governor_mutex);
 429         }
 430   out:
 431         return err;
 432 }
 433
 434
 435 /* drivers/base/cpu.c */
 436 extern struct sysdev_class cpu_sysdev_class;
 437
 438
 439 /**
 440  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 441  * print out cpufreq information
 442  *
 443  * Write out information from cpufreq_driver->policy[cpu]; object must be
 444  * "unsigned int".
 445  */
 446
 447 #define show_one(file_name, object)                     \
 448 static ssize_t show_##file_name                         \
 449 (struct cpufreq_policy * policy, char *buf)             \
 450 {                                                       \
 451         return sprintf (buf, "%u\n", policy->object);   \
 452 }
 453
 454 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 455 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy * policy, const char *buf, size_t count)         \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf (buf, "%u", &new_policy.object);                   \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq,min);
 488 store_one(scaling_max_freq,max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq (struct cpufreq_policy * policy,
 494                                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor (struct cpufreq_policy * policy,
 507                                                         char *buf)
 508 {
 509         if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor (struct cpufreq_policy * policy,
 523                                        const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf (buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver (struct cpufreq_policy * policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors (struct cpufreq_policy *policy,
 566                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 578                         goto out;
 579                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 580         }
 581 out:
 582         i += sprintf(&buf[i], "\n");
 583         return i;
 584 }
 585 /**
 586  * show_affected_cpus - show the CPUs affected by each transition
 587  */
 588 static ssize_t show_affected_cpus (struct cpufreq_policy * policy, char *buf)
 589 {
 590         ssize_t i = 0;
 591         unsigned int cpu;
 592
 593         for_each_cpu_mask(cpu, policy->cpus) {
 594                 if (i)
 595                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 596                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 597                 if (i >= (PAGE_SIZE - 5))
 598                     break;
 599         }
 600         i += sprintf(&buf[i], "\n");
 601         return i;
 602 }
 603
 604 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 605                 const char *buf, size_t count)
 606 {
 607         unsigned int freq = 0;
 608         unsigned int ret;
 609
 610         if (!policy->governor->store_setspeed)
 611                 return -EINVAL;
 612
 613         ret = sscanf(buf, "%u", &freq);
 614         if (ret != 1)
 615                 return -EINVAL;
 616
 617         policy->governor->store_setspeed(policy, freq);
 618
 619         return count;
 620 }
 621
 622 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 623 {
 624         if (!policy->governor->show_setspeed)
 625                 return sprintf(buf, "<unsupported>\n");
 626
 627         return policy->governor->show_setspeed(policy, buf);
 628 }
 629
 630 #define define_one_ro(_name) \
 631 static struct freq_attr _name = \
 632 __ATTR(_name, 0444, show_##_name, NULL)
 633
 634 #define define_one_ro0400(_name) \
 635 static struct freq_attr _name = \
 636 __ATTR(_name, 0400, show_##_name, NULL)
 637
 638 #define define_one_rw(_name) \
 639 static struct freq_attr _name = \
 640 __ATTR(_name, 0644, show_##_name, store_##_name)
 641
 642 define_one_ro0400(cpuinfo_cur_freq);
 643 define_one_ro(cpuinfo_min_freq);
 644 define_one_ro(cpuinfo_max_freq);
 645 define_one_ro(scaling_available_governors);
 646 define_one_ro(scaling_driver);
 647 define_one_ro(scaling_cur_freq);
 648 define_one_ro(affected_cpus);
 649 define_one_rw(scaling_min_freq);
 650 define_one_rw(scaling_max_freq);
 651 define_one_rw(scaling_governor);
 652 define_one_rw(scaling_setspeed);
 653
 654 static struct attribute * default_attrs[] = {
 655         &cpuinfo_min_freq.attr,
 656         &cpuinfo_max_freq.attr,
 657         &scaling_min_freq.attr,
 658         &scaling_max_freq.attr,
 659         &affected_cpus.attr,
 660         &scaling_governor.attr,
 661         &scaling_driver.attr,
 662         &scaling_available_governors.attr,
 663         &scaling_setspeed.attr,
 664         NULL
 665 };
 666
 667 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 668 #define to_attr(a) container_of(a,struct freq_attr,attr)
 669
 670 static ssize_t show(struct kobject * kobj, struct attribute * attr ,char * buf)
 671 {
 672         struct cpufreq_policy * policy = to_policy(kobj);
 673         struct freq_attr * fattr = to_attr(attr);
 674         ssize_t ret;
 675         policy = cpufreq_cpu_get(policy->cpu);
 676         if (!policy)
 677                 return -EINVAL;
 678
 679         if (lock_policy_rwsem_read(policy->cpu) < 0)
 680                 return -EINVAL;
 681
 682         if (fattr->show)
 683                 ret = fattr->show(policy, buf);
 684         else
 685                 ret = -EIO;
 686
 687         unlock_policy_rwsem_read(policy->cpu);
 688
 689         cpufreq_cpu_put(policy);
 690         return ret;
 691 }
 692
 693 static ssize_t store(struct kobject * kobj, struct attribute * attr,
 694                      const char * buf, size_t count)
 695 {
 696         struct cpufreq_policy * policy = to_policy(kobj);
 697         struct freq_attr * fattr = to_attr(attr);
 698         ssize_t ret;
 699         policy = cpufreq_cpu_get(policy->cpu);
 700         if (!policy)
 701                 return -EINVAL;
 702
 703         if (lock_policy_rwsem_write(policy->cpu) < 0)
 704                 return -EINVAL;
 705
 706         if (fattr->store)
 707                 ret = fattr->store(policy, buf, count);
 708         else
 709                 ret = -EIO;
 710
 711         unlock_policy_rwsem_write(policy->cpu);
 712
 713         cpufreq_cpu_put(policy);
 714         return ret;
 715 }
 716
 717 static void cpufreq_sysfs_release(struct kobject * kobj)
 718 {
 719         struct cpufreq_policy * policy = to_policy(kobj);
 720         dprintk("last reference is dropped\n");
 721         complete(&policy->kobj_unregister);
 722 }
 723
 724 static struct sysfs_ops sysfs_ops = {
 725         .show   = show,
 726         .store  = store,
 727 };
 728
 729 static struct kobj_type ktype_cpufreq = {
 730         .sysfs_ops      = &sysfs_ops,
 731         .default_attrs  = default_attrs,
 732         .release        = cpufreq_sysfs_release,
 733 };
 734
 735
 736 /**
 737  * cpufreq_add_dev - add a CPU device
 738  *
 739  * Adds the cpufreq interface for a CPU device.
 740  */
 741 static int cpufreq_add_dev (struct sys_device * sys_dev)
 742 {
 743         unsigned int cpu = sys_dev->id;
 744         int ret = 0;
 745         struct cpufreq_policy new_policy;
 746         struct cpufreq_policy *policy;
 747         struct freq_attr **drv_attr;
 748         struct sys_device *cpu_sys_dev;
 749         unsigned long flags;
 750         unsigned int j;
 751 #ifdef CONFIG_SMP
 752         struct cpufreq_policy *managed_policy;
 753 #endif
 754
 755         if (cpu_is_offline(cpu))
 756                 return 0;
 757
 758         cpufreq_debug_disable_ratelimit();
 759         dprintk("adding CPU %u\n", cpu);
 760
 761 #ifdef CONFIG_SMP
 762         /* check whether a different CPU already registered this
 763          * CPU because it is in the same boat. */
 764         policy = cpufreq_cpu_get(cpu);
 765         if (unlikely(policy)) {
 766                 cpufreq_cpu_put(policy);
 767                 cpufreq_debug_enable_ratelimit();
 768                 return 0;
 769         }
 770 #endif
 771
 772         if (!try_module_get(cpufreq_driver->owner)) {
 773                 ret = -EINVAL;
 774                 goto module_out;
 775         }
 776
 777         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 778         if (!policy) {
 779                 ret = -ENOMEM;
 780                 goto nomem_out;
 781         }
 782
 783         policy->cpu = cpu;
 784         policy->cpus = cpumask_of_cpu(cpu);
 785
 786         /* Initially set CPU itself as the policy_cpu */
 787         per_cpu(policy_cpu, cpu) = cpu;
 788         lock_policy_rwsem_write(cpu);
 789
 790         init_completion(&policy->kobj_unregister);
 791         INIT_WORK(&policy->update, handle_update);
 792
 793         /* Set governor before ->init, so that driver could check it */
 794         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 795         /* call driver. From then on the cpufreq must be able
 796          * to accept all calls to ->verify and ->setpolicy for this CPU
 797          */
 798         ret = cpufreq_driver->init(policy);
 799         if (ret) {
 800                 dprintk("initialization failed\n");
 801                 unlock_policy_rwsem_write(cpu);
 802                 goto err_out;
 803         }
 804         policy->user_policy.min = policy->cpuinfo.min_freq;
 805         policy->user_policy.max = policy->cpuinfo.max_freq;
 806
 807 #ifdef CONFIG_SMP
 808
 809 #ifdef CONFIG_HOTPLUG_CPU
 810         if (cpufreq_cpu_governor[cpu]){
 811                 policy->governor = cpufreq_cpu_governor[cpu];
 812                 dprintk("Restoring governor %s for cpu %d\n",
 813                        policy->governor->name, cpu);
 814         }
 815 #endif
 816
 817         for_each_cpu_mask(j, policy->cpus) {
 818                 if (cpu == j)
 819                         continue;
 820
 821                 /* check for existing affected CPUs.  They may not be aware
 822                  * of it due to CPU Hotplug.
 823                  */
 824                 managed_policy = cpufreq_cpu_get(j);
 825                 if (unlikely(managed_policy)) {
 826
 827                         /* Set proper policy_cpu */
 828                         unlock_policy_rwsem_write(cpu);
 829                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 830
 831                         if (lock_policy_rwsem_write(cpu) < 0)
 832                                 goto err_out_driver_exit;
 833
 834                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 835                         managed_policy->cpus = policy->cpus;
 836                         cpufreq_cpu_data[cpu] = managed_policy;
 837                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 838
 839                         dprintk("CPU already managed, adding link\n");
 840                         ret = sysfs_create_link(&sys_dev->kobj,
 841                                                 &managed_policy->kobj,
 842                                                 "cpufreq");
 843                         if (ret) {
 844                                 unlock_policy_rwsem_write(cpu);
 845                                 goto err_out_driver_exit;
 846                         }
 847
 848                         cpufreq_debug_enable_ratelimit();
 849                         ret = 0;
 850                         unlock_policy_rwsem_write(cpu);
 851                         goto err_out_driver_exit; /* call driver->exit() */
 852                 }
 853         }
 854 #endif
 855         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 856
 857         /* prepare interface data */
 858         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 859                                    "cpufreq");
 860         if (ret) {
 861                 unlock_policy_rwsem_write(cpu);
 862                 goto err_out_driver_exit;
 863         }
 864         /* set up files for this cpu device */
 865         drv_attr = cpufreq_driver->attr;
 866         while ((drv_attr) && (*drv_attr)) {
 867                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 868                 if (ret) {
 869                         unlock_policy_rwsem_write(cpu);
 870                         goto err_out_driver_exit;
 871                 }
 872                 drv_attr++;
 873         }
 874         if (cpufreq_driver->get){
 875                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 876                 if (ret) {
 877                         unlock_policy_rwsem_write(cpu);
 878                         goto err_out_driver_exit;
 879                 }
 880         }
 881         if (cpufreq_driver->target){
 882                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 883                 if (ret) {
 884                         unlock_policy_rwsem_write(cpu);
 885                         goto err_out_driver_exit;
 886                 }
 887         }
 888
 889         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 890         for_each_cpu_mask(j, policy->cpus) {
 891                 cpufreq_cpu_data[j] = policy;
 892                 per_cpu(policy_cpu, j) = policy->cpu;
 893         }
 894         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 895
 896         /* symlink affected CPUs */
 897         for_each_cpu_mask(j, policy->cpus) {
 898                 if (j == cpu)
 899                         continue;
 900                 if (!cpu_online(j))
 901                         continue;
 902
 903                 dprintk("CPU %u already managed, adding link\n", j);
 904                 cpufreq_cpu_get(cpu);
 905                 cpu_sys_dev = get_cpu_sysdev(j);
 906                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 907                                         "cpufreq");
 908                 if (ret) {
 909                         unlock_policy_rwsem_write(cpu);
 910                         goto err_out_unregister;
 911                 }
 912         }
 913
 914         policy->governor = NULL; /* to assure that the starting sequence is
 915                                   * run in cpufreq_set_policy */
 916
 917         /* set default policy */
 918         ret = __cpufreq_set_policy(policy, &new_policy);
 919         policy->user_policy.policy = policy->policy;
 920         policy->user_policy.governor = policy->governor;
 921
 922         unlock_policy_rwsem_write(cpu);
 923
 924         if (ret) {
 925                 dprintk("setting policy failed\n");
 926                 goto err_out_unregister;
 927         }
 928
 929         kobject_uevent(&policy->kobj, KOBJ_ADD);
 930         module_put(cpufreq_driver->owner);
 931         dprintk("initialization complete\n");
 932         cpufreq_debug_enable_ratelimit();
 933
 934         return 0;
 935
 936
 937 err_out_unregister:
 938         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 939         for_each_cpu_mask(j, policy->cpus)
 940                 cpufreq_cpu_data[j] = NULL;
 941         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 942
 943         kobject_put(&policy->kobj);
 944         wait_for_completion(&policy->kobj_unregister);
 945
 946 err_out_driver_exit:
 947         if (cpufreq_driver->exit)
 948                 cpufreq_driver->exit(policy);
 949
 950 err_out:
 951         kfree(policy);
 952
 953 nomem_out:
 954         module_put(cpufreq_driver->owner);
 955 module_out:
 956         cpufreq_debug_enable_ratelimit();
 957         return ret;
 958 }
 959
 960
 961 /**
 962  * __cpufreq_remove_dev - remove a CPU device
 963  *
 964  * Removes the cpufreq interface for a CPU device.
 965  * Caller should already have policy_rwsem in write mode for this CPU.
 966  * This routine frees the rwsem before returning.
 967  */
 968 static int __cpufreq_remove_dev (struct sys_device * sys_dev)
 969 {
 970         unsigned int cpu = sys_dev->id;
 971         unsigned long flags;
 972         struct cpufreq_policy *data;
 973 #ifdef CONFIG_SMP
 974         struct sys_device *cpu_sys_dev;
 975         unsigned int j;
 976 #endif
 977
 978         cpufreq_debug_disable_ratelimit();
 979         dprintk("unregistering CPU %u\n", cpu);
 980
 981         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 982         data = cpufreq_cpu_data[cpu];
 983
 984         if (!data) {
 985                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 986                 cpufreq_debug_enable_ratelimit();
 987                 unlock_policy_rwsem_write(cpu);
 988                 return -EINVAL;
 989         }
 990         cpufreq_cpu_data[cpu] = NULL;
 991
 992
 993 #ifdef CONFIG_SMP
 994         /* if this isn't the CPU which is the parent of the kobj, we
 995          * only need to unlink, put and exit
 996          */
 997         if (unlikely(cpu != data->cpu)) {
 998                 dprintk("removing link\n");
 999                 cpu_clear(cpu, data->cpus);
1000                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1001                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1002                 cpufreq_cpu_put(data);
1003                 cpufreq_debug_enable_ratelimit();
1004                 unlock_policy_rwsem_write(cpu);
1005                 return 0;
1006         }
1007 #endif
1008
1009
1010         if (!kobject_get(&data->kobj)) {
1011                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1012                 cpufreq_debug_enable_ratelimit();
1013                 unlock_policy_rwsem_write(cpu);
1014                 return -EFAULT;
1015         }
1016
1017 #ifdef CONFIG_SMP
1018
1019 #ifdef CONFIG_HOTPLUG_CPU
1020         cpufreq_cpu_governor[cpu] = data->governor;
1021 #endif
1022
1023         /* if we have other CPUs still registered, we need to unlink them,
1024          * or else wait_for_completion below will lock up. Clean the
1025          * cpufreq_cpu_data[] while holding the lock, and remove the sysfs
1026          * links afterwards.
1027          */
1028         if (unlikely(cpus_weight(data->cpus) > 1)) {
1029                 for_each_cpu_mask(j, data->cpus) {
1030                         if (j == cpu)
1031                                 continue;
1032                         cpufreq_cpu_data[j] = NULL;
1033                 }
1034         }
1035
1036         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1037
1038         if (unlikely(cpus_weight(data->cpus) > 1)) {
1039                 for_each_cpu_mask(j, data->cpus) {
1040                         if (j == cpu)
1041                                 continue;
1042                         dprintk("removing link for cpu %u\n", j);
1043 #ifdef CONFIG_HOTPLUG_CPU
1044                         cpufreq_cpu_governor[j] = data->governor;
1045 #endif
1046                         cpu_sys_dev = get_cpu_sysdev(j);
1047                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1048                         cpufreq_cpu_put(data);
1049                 }
1050         }
1051 #else
1052         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1053 #endif
1054
1055         if (cpufreq_driver->target)
1056                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1057
1058         unlock_policy_rwsem_write(cpu);
1059
1060         kobject_put(&data->kobj);
1061
1062         /* we need to make sure that the underlying kobj is actually
1063          * not referenced anymore by anybody before we proceed with
1064          * unloading.
1065          */
1066         dprintk("waiting for dropping of refcount\n");
1067         wait_for_completion(&data->kobj_unregister);
1068         dprintk("wait complete\n");
1069
1070         if (cpufreq_driver->exit)
1071                 cpufreq_driver->exit(data);
1072
1073         kfree(data);
1074
1075         cpufreq_debug_enable_ratelimit();
1076         return 0;
1077 }
1078
1079
1080 static int cpufreq_remove_dev (struct sys_device * sys_dev)
1081 {
1082         unsigned int cpu = sys_dev->id;
1083         int retval;
1084
1085         if (cpu_is_offline(cpu))
1086                 return 0;
1087
1088         if (unlikely(lock_policy_rwsem_write(cpu)))
1089                 BUG();
1090
1091         retval = __cpufreq_remove_dev(sys_dev);
1092         return retval;
1093 }
1094
1095
1096 static void handle_update(struct work_struct *work)
1097 {
1098         struct cpufreq_policy *policy =
1099                 container_of(work, struct cpufreq_policy, update);
1100         unsigned int cpu = policy->cpu;
1101         dprintk("handle_update for cpu %u called\n", cpu);
1102         cpufreq_update_policy(cpu);
1103 }
1104
1105 /**
1106  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1107  *      @cpu: cpu number
1108  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1109  *      @new_freq: CPU frequency the CPU actually runs at
1110  *
1111  *      We adjust to current frequency first, and need to clean up later. So either call
1112  *      to cpufreq_update_policy() or schedule handle_update()).
1113  */
1114 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1115                                 unsigned int new_freq)
1116 {
1117         struct cpufreq_freqs freqs;
1118
1119         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1120                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1121
1122         freqs.cpu = cpu;
1123         freqs.old = old_freq;
1124         freqs.new = new_freq;
1125         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1126         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1127 }
1128
1129
1130 /**
1131  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1132  * @cpu: CPU number
1133  *
1134  * This is the last known freq, without actually getting it from the driver.
1135  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1136  */
1137 unsigned int cpufreq_quick_get(unsigned int cpu)
1138 {
1139         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1140         unsigned int ret_freq = 0;
1141
1142         if (policy) {
1143                 ret_freq = policy->cur;
1144                 cpufreq_cpu_put(policy);
1145         }
1146
1147         return (ret_freq);
1148 }
1149 EXPORT_SYMBOL(cpufreq_quick_get);
1150
1151
1152 static unsigned int __cpufreq_get(unsigned int cpu)
1153 {
1154         struct cpufreq_policy *policy = cpufreq_cpu_data[cpu];
1155         unsigned int ret_freq = 0;
1156
1157         if (!cpufreq_driver->get)
1158                 return (ret_freq);
1159
1160         ret_freq = cpufreq_driver->get(cpu);
1161
1162         if (ret_freq && policy->cur &&
1163                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1164                 /* verify no discrepancy between actual and
1165                                         saved value exists */
1166                 if (unlikely(ret_freq != policy->cur)) {
1167                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1168                         schedule_work(&policy->update);
1169                 }
1170         }
1171
1172         return (ret_freq);
1173 }
1174
1175 /**
1176  * cpufreq_get - get the current CPU frequency (in kHz)
1177  * @cpu: CPU number
1178  *
1179  * Get the CPU current (static) CPU frequency
1180  */
1181 unsigned int cpufreq_get(unsigned int cpu)
1182 {
1183         unsigned int ret_freq = 0;
1184         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1185
1186         if (!policy)
1187                 goto out;
1188
1189         if (unlikely(lock_policy_rwsem_read(cpu)))
1190                 goto out_policy;
1191
1192         ret_freq = __cpufreq_get(cpu);
1193
1194         unlock_policy_rwsem_read(cpu);
1195
1196 out_policy:
1197         cpufreq_cpu_put(policy);
1198 out:
1199         return (ret_freq);
1200 }
1201 EXPORT_SYMBOL(cpufreq_get);
1202
1203
1204 /**
1205  *      cpufreq_suspend - let the low level driver prepare for suspend
1206  */
1207
1208 static int cpufreq_suspend(struct sys_device * sysdev, pm_message_t pmsg)
1209 {
1210         int cpu = sysdev->id;
1211         int ret = 0;
1212         unsigned int cur_freq = 0;
1213         struct cpufreq_policy *cpu_policy;
1214
1215         dprintk("suspending cpu %u\n", cpu);
1216
1217         if (!cpu_online(cpu))
1218                 return 0;
1219
1220         /* we may be lax here as interrupts are off. Nonetheless
1221          * we need to grab the correct cpu policy, as to check
1222          * whether we really run on this CPU.
1223          */
1224
1225         cpu_policy = cpufreq_cpu_get(cpu);
1226         if (!cpu_policy)
1227                 return -EINVAL;
1228
1229         /* only handle each CPU group once */
1230         if (unlikely(cpu_policy->cpu != cpu)) {
1231                 cpufreq_cpu_put(cpu_policy);
1232                 return 0;
1233         }
1234
1235         if (cpufreq_driver->suspend) {
1236                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1237                 if (ret) {
1238                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1239                                         "step on CPU %u\n", cpu_policy->cpu);
1240                         cpufreq_cpu_put(cpu_policy);
1241                         return ret;
1242                 }
1243         }
1244
1245
1246         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1247                 goto out;
1248
1249         if (cpufreq_driver->get)
1250                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1251
1252         if (!cur_freq || !cpu_policy->cur) {
1253                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1254                        "frequency is what timing core thinks it is.\n");
1255                 goto out;
1256         }
1257
1258         if (unlikely(cur_freq != cpu_policy->cur)) {
1259                 struct cpufreq_freqs freqs;
1260
1261                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1262                         dprintk("Warning: CPU frequency is %u, "
1263                                "cpufreq assumed %u kHz.\n",
1264                                cur_freq, cpu_policy->cur);
1265
1266                 freqs.cpu = cpu;
1267                 freqs.old = cpu_policy->cur;
1268                 freqs.new = cur_freq;
1269
1270                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1271                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1272                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1273
1274                 cpu_policy->cur = cur_freq;
1275         }
1276
1277 out:
1278         cpufreq_cpu_put(cpu_policy);
1279         return 0;
1280 }
1281
1282 /**
1283  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1284  *
1285  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1286  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1287  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1288  *          restored.
1289  */
1290 static int cpufreq_resume(struct sys_device * sysdev)
1291 {
1292         int cpu = sysdev->id;
1293         int ret = 0;
1294         struct cpufreq_policy *cpu_policy;
1295
1296         dprintk("resuming cpu %u\n", cpu);
1297
1298         if (!cpu_online(cpu))
1299                 return 0;
1300
1301         /* we may be lax here as interrupts are off. Nonetheless
1302          * we need to grab the correct cpu policy, as to check
1303          * whether we really run on this CPU.
1304          */
1305
1306         cpu_policy = cpufreq_cpu_get(cpu);
1307         if (!cpu_policy)
1308                 return -EINVAL;
1309
1310         /* only handle each CPU group once */
1311         if (unlikely(cpu_policy->cpu != cpu)) {
1312                 cpufreq_cpu_put(cpu_policy);
1313                 return 0;
1314         }
1315
1316         if (cpufreq_driver->resume) {
1317                 ret = cpufreq_driver->resume(cpu_policy);
1318                 if (ret) {
1319                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1320                                         "step on CPU %u\n", cpu_policy->cpu);
1321                         cpufreq_cpu_put(cpu_policy);
1322                         return ret;
1323                 }
1324         }
1325
1326         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1327                 unsigned int cur_freq = 0;
1328
1329                 if (cpufreq_driver->get)
1330                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1331
1332                 if (!cur_freq || !cpu_policy->cur) {
1333                         printk(KERN_ERR "cpufreq: resume failed to assert "
1334                                         "current frequency is what timing core "
1335                                         "thinks it is.\n");
1336                         goto out;
1337                 }
1338
1339                 if (unlikely(cur_freq != cpu_policy->cur)) {
1340                         struct cpufreq_freqs freqs;
1341
1342                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1343                                 dprintk("Warning: CPU frequency "
1344                                        "is %u, cpufreq assumed %u kHz.\n",
1345                                        cur_freq, cpu_policy->cur);
1346
1347                         freqs.cpu = cpu;
1348                         freqs.old = cpu_policy->cur;
1349                         freqs.new = cur_freq;
1350
1351                         srcu_notifier_call_chain(
1352                                         &cpufreq_transition_notifier_list,
1353                                         CPUFREQ_RESUMECHANGE, &freqs);
1354                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1355
1356                         cpu_policy->cur = cur_freq;
1357                 }
1358         }
1359
1360 out:
1361         schedule_work(&cpu_policy->update);
1362         cpufreq_cpu_put(cpu_policy);
1363         return ret;
1364 }
1365
1366 static struct sysdev_driver cpufreq_sysdev_driver = {
1367         .add            = cpufreq_add_dev,
1368         .remove         = cpufreq_remove_dev,
1369         .suspend        = cpufreq_suspend,
1370         .resume         = cpufreq_resume,
1371 };
1372
1373
1374 /*********************************************************************
1375  *                     NOTIFIER LISTS INTERFACE                      *
1376  *********************************************************************/
1377
1378 /**
1379  *      cpufreq_register_notifier - register a driver with cpufreq
1380  *      @nb: notifier function to register
1381  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1382  *
1383  *      Add a driver to one of two lists: either a list of drivers that
1384  *      are notified about clock rate changes (once before and once after
1385  *      the transition), or a list of drivers that are notified about
1386  *      changes in cpufreq policy.
1387  *
1388  *      This function may sleep, and has the same return conditions as
1389  *      blocking_notifier_chain_register.
1390  */
1391 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1392 {
1393         int ret;
1394
1395         switch (list) {
1396         case CPUFREQ_TRANSITION_NOTIFIER:
1397                 ret = srcu_notifier_chain_register(
1398                                 &cpufreq_transition_notifier_list, nb);
1399                 break;
1400         case CPUFREQ_POLICY_NOTIFIER:
1401                 ret = blocking_notifier_chain_register(
1402                                 &cpufreq_policy_notifier_list, nb);
1403                 break;
1404         default:
1405                 ret = -EINVAL;
1406         }
1407
1408         return ret;
1409 }
1410 EXPORT_SYMBOL(cpufreq_register_notifier);
1411
1412
1413 /**
1414  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1415  *      @nb: notifier block to be unregistered
1416  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1417  *
1418  *      Remove a driver from the CPU frequency notifier list.
1419  *
1420  *      This function may sleep, and has the same return conditions as
1421  *      blocking_notifier_chain_unregister.
1422  */
1423 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1424 {
1425         int ret;
1426
1427         switch (list) {
1428         case CPUFREQ_TRANSITION_NOTIFIER:
1429                 ret = srcu_notifier_chain_unregister(
1430                                 &cpufreq_transition_notifier_list, nb);
1431                 break;
1432         case CPUFREQ_POLICY_NOTIFIER:
1433                 ret = blocking_notifier_chain_unregister(
1434                                 &cpufreq_policy_notifier_list, nb);
1435                 break;
1436         default:
1437                 ret = -EINVAL;
1438         }
1439
1440         return ret;
1441 }
1442 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1443
1444
1445 /*********************************************************************
1446  *                              GOVERNORS                            *
1447  *********************************************************************/
1448
1449
1450 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1451                             unsigned int target_freq,
1452                             unsigned int relation)
1453 {
1454         int retval = -EINVAL;
1455
1456         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1457                 target_freq, relation);
1458         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1459                 retval = cpufreq_driver->target(policy, target_freq, relation);
1460
1461         return retval;
1462 }
1463 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1464
1465 int cpufreq_driver_target(struct cpufreq_policy *policy,
1466                           unsigned int target_freq,
1467                           unsigned int relation)
1468 {
1469         int ret;
1470
1471         policy = cpufreq_cpu_get(policy->cpu);
1472         if (!policy)
1473                 return -EINVAL;
1474
1475         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1476                 return -EINVAL;
1477
1478         ret = __cpufreq_driver_target(policy, target_freq, relation);
1479
1480         unlock_policy_rwsem_write(policy->cpu);
1481
1482         cpufreq_cpu_put(policy);
1483         return ret;
1484 }
1485 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1486
1487 int __cpufreq_driver_getavg(struct cpufreq_policy *policy)
1488 {
1489         int ret = 0;
1490
1491         policy = cpufreq_cpu_get(policy->cpu);
1492         if (!policy)
1493                 return -EINVAL;
1494
1495         if (cpu_online(policy->cpu) && cpufreq_driver->getavg)
1496                 ret = cpufreq_driver->getavg(policy->cpu);
1497
1498         cpufreq_cpu_put(policy);
1499         return ret;
1500 }
1501 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1502
1503 /*
1504  * when "event" is CPUFREQ_GOV_LIMITS
1505  */
1506
1507 static int __cpufreq_governor(struct cpufreq_policy *policy,
1508                                         unsigned int event)
1509 {
1510         int ret;
1511
1512         /* Only must be defined when default governor is known to have latency
1513            restrictions, like e.g. conservative or ondemand.
1514            That this is the case is already ensured in Kconfig
1515         */
1516 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1517         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1518 #else
1519         struct cpufreq_governor *gov = NULL;
1520 #endif
1521
1522         if (policy->governor->max_transition_latency &&
1523             policy->cpuinfo.transition_latency >
1524             policy->governor->max_transition_latency) {
1525                 if (!gov)
1526                         return -EINVAL;
1527                 else {
1528                         printk(KERN_WARNING "%s governor failed, too long"
1529                                " transition latency of HW, fallback"
1530                                " to %s governor\n",
1531                                policy->governor->name,
1532                                gov->name);
1533                         policy->governor = gov;
1534                 }
1535         }
1536
1537         if (!try_module_get(policy->governor->owner))
1538                 return -EINVAL;
1539
1540         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1541                                                 policy->cpu, event);
1542         ret = policy->governor->governor(policy, event);
1543
1544         /* we keep one module reference alive for
1545                         each CPU governed by this CPU */
1546         if ((event != CPUFREQ_GOV_START) || ret)
1547                 module_put(policy->governor->owner);
1548         if ((event == CPUFREQ_GOV_STOP) && !ret)
1549                 module_put(policy->governor->owner);
1550
1551         return ret;
1552 }
1553
1554
1555 int cpufreq_register_governor(struct cpufreq_governor *governor)
1556 {
1557         int err;
1558
1559         if (!governor)
1560                 return -EINVAL;
1561
1562         mutex_lock(&cpufreq_governor_mutex);
1563
1564         err = -EBUSY;
1565         if (__find_governor(governor->name) == NULL) {
1566                 err = 0;
1567                 list_add(&governor->governor_list, &cpufreq_governor_list);
1568         }
1569
1570         mutex_unlock(&cpufreq_governor_mutex);
1571         return err;
1572 }
1573 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1574
1575
1576 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1577 {
1578         if (!governor)
1579                 return;
1580
1581         mutex_lock(&cpufreq_governor_mutex);
1582         list_del(&governor->governor_list);
1583         mutex_unlock(&cpufreq_governor_mutex);
1584         return;
1585 }
1586 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1587
1588
1589
1590 /*********************************************************************
1591  *                          POLICY INTERFACE                         *
1592  *********************************************************************/
1593
1594 /**
1595  * cpufreq_get_policy - get the current cpufreq_policy
1596  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1597  *
1598  * Reads the current cpufreq policy.
1599  */
1600 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1601 {
1602         struct cpufreq_policy *cpu_policy;
1603         if (!policy)
1604                 return -EINVAL;
1605
1606         cpu_policy = cpufreq_cpu_get(cpu);
1607         if (!cpu_policy)
1608                 return -EINVAL;
1609
1610         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1611
1612         cpufreq_cpu_put(cpu_policy);
1613         return 0;
1614 }
1615 EXPORT_SYMBOL(cpufreq_get_policy);
1616
1617
1618 /*
1619  * data   : current policy.
1620  * policy : policy to be set.
1621  */
1622 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1623                                 struct cpufreq_policy *policy)
1624 {
1625         int ret = 0;
1626
1627         cpufreq_debug_disable_ratelimit();
1628         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1629                 policy->min, policy->max);
1630
1631         memcpy(&policy->cpuinfo, &data->cpuinfo,
1632                                 sizeof(struct cpufreq_cpuinfo));
1633
1634         if (policy->min > data->max || policy->max < data->min) {
1635                 ret = -EINVAL;
1636                 goto error_out;
1637         }
1638
1639         /* verify the cpu speed can be set within this limit */
1640         ret = cpufreq_driver->verify(policy);
1641         if (ret)
1642                 goto error_out;
1643
1644         /* adjust if necessary - all reasons */
1645         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1646                         CPUFREQ_ADJUST, policy);
1647
1648         /* adjust if necessary - hardware incompatibility*/
1649         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1650                         CPUFREQ_INCOMPATIBLE, policy);
1651
1652         /* verify the cpu speed can be set within this limit,
1653            which might be different to the first one */
1654         ret = cpufreq_driver->verify(policy);
1655         if (ret)
1656                 goto error_out;
1657
1658         /* notification of the new policy */
1659         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1660                         CPUFREQ_NOTIFY, policy);
1661
1662         data->min = policy->min;
1663         data->max = policy->max;
1664
1665         dprintk("new min and max freqs are %u - %u kHz\n",
1666                                         data->min, data->max);
1667
1668         if (cpufreq_driver->setpolicy) {
1669                 data->policy = policy->policy;
1670                 dprintk("setting range\n");
1671                 ret = cpufreq_driver->setpolicy(policy);
1672         } else {
1673                 if (policy->governor != data->governor) {
1674                         /* save old, working values */
1675                         struct cpufreq_governor *old_gov = data->governor;
1676
1677                         dprintk("governor switch\n");
1678
1679                         /* end old governor */
1680                         if (data->governor)
1681                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1682
1683                         /* start new governor */
1684                         data->governor = policy->governor;
1685                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1686                                 /* new governor failed, so re-start old one */
1687                                 dprintk("starting governor %s failed\n",
1688                                                         data->governor->name);
1689                                 if (old_gov) {
1690                                         data->governor = old_gov;
1691                                         __cpufreq_governor(data,
1692                                                            CPUFREQ_GOV_START);
1693                                 }
1694                                 ret = -EINVAL;
1695                                 goto error_out;
1696                         }
1697                         /* might be a policy change, too, so fall through */
1698                 }
1699                 dprintk("governor: change or update limits\n");
1700                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1701         }
1702
1703 error_out:
1704         cpufreq_debug_enable_ratelimit();
1705         return ret;
1706 }
1707
1708 /**
1709  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1710  *      @cpu: CPU which shall be re-evaluated
1711  *
1712  *      Usefull for policy notifiers which have different necessities
1713  *      at different times.
1714  */
1715 int cpufreq_update_policy(unsigned int cpu)
1716 {
1717         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1718         struct cpufreq_policy policy;
1719         int ret = 0;
1720
1721         if (!data)
1722                 return -ENODEV;
1723
1724         if (unlikely(lock_policy_rwsem_write(cpu)))
1725                 return -EINVAL;
1726
1727         dprintk("updating policy for CPU %u\n", cpu);
1728         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1729         policy.min = data->user_policy.min;
1730         policy.max = data->user_policy.max;
1731         policy.policy = data->user_policy.policy;
1732         policy.governor = data->user_policy.governor;
1733
1734         /* BIOS might change freq behind our back
1735           -> ask driver for current freq and notify governors about a change */
1736         if (cpufreq_driver->get) {
1737                 policy.cur = cpufreq_driver->get(cpu);
1738                 if (!data->cur) {
1739                         dprintk("Driver did not initialize current freq");
1740                         data->cur = policy.cur;
1741                 } else {
1742                         if (data->cur != policy.cur)
1743                                 cpufreq_out_of_sync(cpu, data->cur,
1744                                                                 policy.cur);
1745                 }
1746         }
1747
1748         ret = __cpufreq_set_policy(data, &policy);
1749
1750         unlock_policy_rwsem_write(cpu);
1751
1752         cpufreq_cpu_put(data);
1753         return ret;
1754 }
1755 EXPORT_SYMBOL(cpufreq_update_policy);
1756
1757 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1758                                         unsigned long action, void *hcpu)
1759 {
1760         unsigned int cpu = (unsigned long)hcpu;
1761         struct sys_device *sys_dev;
1762
1763         sys_dev = get_cpu_sysdev(cpu);
1764         if (sys_dev) {
1765                 switch (action) {
1766                 case CPU_ONLINE:
1767                 case CPU_ONLINE_FROZEN:
1768                         cpufreq_add_dev(sys_dev);
1769                         break;
1770                 case CPU_DOWN_PREPARE:
1771                 case CPU_DOWN_PREPARE_FROZEN:
1772                         if (unlikely(lock_policy_rwsem_write(cpu)))
1773                                 BUG();
1774
1775                         __cpufreq_remove_dev(sys_dev);
1776                         break;
1777                 case CPU_DOWN_FAILED:
1778                 case CPU_DOWN_FAILED_FROZEN:
1779                         cpufreq_add_dev(sys_dev);
1780                         break;
1781                 }
1782         }
1783         return NOTIFY_OK;
1784 }
1785
1786 static struct notifier_block __cpuinitdata cpufreq_cpu_notifier =
1787 {
1788     .notifier_call = cpufreq_cpu_callback,
1789 };
1790
1791 /*********************************************************************
1792  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1793  *********************************************************************/
1794
1795 /**
1796  * cpufreq_register_driver - register a CPU Frequency driver
1797  * @driver_data: A struct cpufreq_driver containing the values#
1798  * submitted by the CPU Frequency driver.
1799  *
1800  *   Registers a CPU Frequency driver to this core code. This code
1801  * returns zero on success, -EBUSY when another driver got here first
1802  * (and isn't unregistered in the meantime).
1803  *
1804  */
1805 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1806 {
1807         unsigned long flags;
1808         int ret;
1809
1810         if (!driver_data || !driver_data->verify || !driver_data->init ||
1811             ((!driver_data->setpolicy) && (!driver_data->target)))
1812                 return -EINVAL;
1813
1814         dprintk("trying to register driver %s\n", driver_data->name);
1815
1816         if (driver_data->setpolicy)
1817                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1818
1819         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1820         if (cpufreq_driver) {
1821                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1822                 return -EBUSY;
1823         }
1824         cpufreq_driver = driver_data;
1825         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1826
1827         ret = sysdev_driver_register(&cpu_sysdev_class,&cpufreq_sysdev_driver);
1828
1829         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1830                 int i;
1831                 ret = -ENODEV;
1832
1833                 /* check for at least one working CPU */
1834                 for (i=0; i<NR_CPUS; i++)
1835                         if (cpufreq_cpu_data[i])
1836                                 ret = 0;
1837
1838                 /* if all ->init() calls failed, unregister */
1839                 if (ret) {
1840                         dprintk("no CPU initialized for driver %s\n",
1841                                                         driver_data->name);
1842                         sysdev_driver_unregister(&cpu_sysdev_class,
1843                                                 &cpufreq_sysdev_driver);
1844
1845                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1846                         cpufreq_driver = NULL;
1847                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1848                 }
1849         }
1850
1851         if (!ret) {
1852                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1853                 dprintk("driver %s up and running\n", driver_data->name);
1854                 cpufreq_debug_enable_ratelimit();
1855         }
1856
1857         return (ret);
1858 }
1859 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1860
1861
1862 /**
1863  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1864  *
1865  *    Unregister the current CPUFreq driver. Only call this if you have
1866  * the right to do so, i.e. if you have succeeded in initialising before!
1867  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1868  * currently not initialised.
1869  */
1870 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1871 {
1872         unsigned long flags;
1873
1874         cpufreq_debug_disable_ratelimit();
1875
1876         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1877                 cpufreq_debug_enable_ratelimit();
1878                 return -EINVAL;
1879         }
1880
1881         dprintk("unregistering driver %s\n", driver->name);
1882
1883         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1884         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1885
1886         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1887         cpufreq_driver = NULL;
1888         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1889
1890         return 0;
1891 }
1892 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1893
1894 static int __init cpufreq_core_init(void)
1895 {
1896         int cpu;
1897
1898         for_each_possible_cpu(cpu) {
1899                 per_cpu(policy_cpu, cpu) = -1;
1900                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1901         }
1902         return 0;
1903 }
1904
1905 core_initcall(cpufreq_core_init);