drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  * - Lock should not be held across
  65  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  66  */
  67 static DEFINE_PER_CPU(int, policy_cpu);
  68 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  69
  70 #define lock_policy_rwsem(mode, cpu)                                    \
  71 int lock_policy_rwsem_##mode                                            \
  72 (int cpu)                                                               \
  73 {                                                                       \
  74         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  75         BUG_ON(policy_cpu == -1);                                       \
  76         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  77         if (unlikely(!cpu_online(cpu))) {                               \
  78                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  79                 return -1;                                              \
  80         }                                                               \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  87
  88 lock_policy_rwsem(write, cpu);
  89 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  90
  91 void unlock_policy_rwsem_read(int cpu)
  92 {
  93         int policy_cpu = per_cpu(policy_cpu, cpu);
  94         BUG_ON(policy_cpu == -1);
  95         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  96 }
  97 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  98
  99 void unlock_policy_rwsem_write(int cpu)
 100 {
 101         int policy_cpu = per_cpu(policy_cpu, cpu);
 102         BUG_ON(policy_cpu == -1);
 103         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 104 }
 105 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 106
 107
 108 /* internal prototypes */
 109 static int __cpufreq_governor(struct cpufreq_policy *policy,
 110                 unsigned int event);
 111 static unsigned int __cpufreq_get(unsigned int cpu);
 112 static void handle_update(struct work_struct *work);
 113
 114 /**
 115  * Two notifier lists: the "policy" list is involved in the
 116  * validation process for a new CPU frequency policy; the
 117  * "transition" list for kernel code that needs to handle
 118  * changes to devices when the CPU clock speed changes.
 119  * The mutex locks both lists.
 120  */
 121 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 122 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 123
 124 static bool init_cpufreq_transition_notifier_list_called;
 125 static int __init init_cpufreq_transition_notifier_list(void)
 126 {
 127         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 128         init_cpufreq_transition_notifier_list_called = true;
 129         return 0;
 130 }
 131 pure_initcall(init_cpufreq_transition_notifier_list);
 132
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 137 {
 138         struct cpufreq_policy *data;
 139         unsigned long flags;
 140
 141         if (cpu >= nr_cpu_ids)
 142                 goto err_out;
 143
 144         /* get the cpufreq driver */
 145         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 146
 147         if (!cpufreq_driver)
 148                 goto err_out_unlock;
 149
 150         if (!try_module_get(cpufreq_driver->owner))
 151                 goto err_out_unlock;
 152
 153
 154         /* get the CPU */
 155         data = per_cpu(cpufreq_cpu_data, cpu);
 156
 157         if (!data)
 158                 goto err_out_put_module;
 159
 160         if (!kobject_get(&data->kobj))
 161                 goto err_out_put_module;
 162
 163         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 164         return data;
 165
 166 err_out_put_module:
 167         module_put(cpufreq_driver->owner);
 168 err_out_unlock:
 169         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 170 err_out:
 171         return NULL;
 172 }
 173 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 174
 175
 176 void cpufreq_cpu_put(struct cpufreq_policy *data)
 177 {
 178         kobject_put(&data->kobj);
 179         module_put(cpufreq_driver->owner);
 180 }
 181 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 182
 183
 184 /*********************************************************************
 185  *                     UNIFIED DEBUG HELPERS                         *
 186  *********************************************************************/
 187 #ifdef CONFIG_CPU_FREQ_DEBUG
 188
 189 /* what part(s) of the CPUfreq subsystem are debugged? */
 190 static unsigned int debug;
 191
 192 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 193  * set or modify this value.
 194  */
 195 static unsigned int debug_ratelimit = 1;
 196
 197 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 198  * loading of a cpufreq driver, temporarily disabled when a new policy
 199  * is set, and disabled upon cpufreq driver removal
 200  */
 201 static unsigned int disable_ratelimit = 1;
 202 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 203
 204 static void cpufreq_debug_enable_ratelimit(void)
 205 {
 206         unsigned long flags;
 207
 208         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 209         if (disable_ratelimit)
 210                 disable_ratelimit--;
 211         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 212 }
 213
 214 static void cpufreq_debug_disable_ratelimit(void)
 215 {
 216         unsigned long flags;
 217
 218         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 219         disable_ratelimit++;
 220         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 221 }
 222
 223 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 224                         const char *fmt, ...)
 225 {
 226         char s[256];
 227         va_list args;
 228         unsigned int len;
 229         unsigned long flags;
 230
 231         WARN_ON(!prefix);
 232         if (type & debug) {
 233                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 234                 if (!disable_ratelimit && debug_ratelimit
 235                                         && !printk_ratelimit()) {
 236                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 237                         return;
 238                 }
 239                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 240
 241                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 242
 243                 va_start(args, fmt);
 244                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 245                 va_end(args);
 246
 247                 printk(s);
 248
 249                 WARN_ON(len < 5);
 250         }
 251 }
 252 EXPORT_SYMBOL(cpufreq_debug_printk);
 253
 254
 255 module_param(debug, uint, 0644);
 256 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 257                         " 2 to debug drivers, and 4 to debug governors.");
 258
 259 module_param(debug_ratelimit, uint, 0644);
 260 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 261                                         " set to 0 to disable ratelimiting.");
 262
 263 #else /* !CONFIG_CPU_FREQ_DEBUG */
 264
 265 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 266 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 267
 268 #endif /* CONFIG_CPU_FREQ_DEBUG */
 269
 270
 271 /*********************************************************************
 272  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 273  *********************************************************************/
 274
 275 /**
 276  * adjust_jiffies - adjust the system "loops_per_jiffy"
 277  *
 278  * This function alters the system "loops_per_jiffy" for the clock
 279  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 280  * systems as each CPU might be scaled differently. So, use the arch
 281  * per-CPU loops_per_jiffy value wherever possible.
 282  */
 283 #ifndef CONFIG_SMP
 284 static unsigned long l_p_j_ref;
 285 static unsigned int  l_p_j_ref_freq;
 286
 287 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 288 {
 289         if (ci->flags & CPUFREQ_CONST_LOOPS)
 290                 return;
 291
 292         if (!l_p_j_ref_freq) {
 293                 l_p_j_ref = loops_per_jiffy;
 294                 l_p_j_ref_freq = ci->old;
 295                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 296                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 297         }
 298         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 299             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 300             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 301                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 302                                                                 ci->new);
 303                 dprintk("scaling loops_per_jiffy to %lu "
 304                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 305         }
 306 }
 307 #else
 308 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 309 {
 310         return;
 311 }
 312 #endif
 313
 314
 315 /**
 316  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 317  * on frequency transition.
 318  *
 319  * This function calls the transition notifiers and the "adjust_jiffies"
 320  * function. It is called twice on all CPU frequency changes that have
 321  * external effects.
 322  */
 323 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 324 {
 325         struct cpufreq_policy *policy;
 326
 327         BUG_ON(irqs_disabled());
 328
 329         freqs->flags = cpufreq_driver->flags;
 330         dprintk("notification %u of frequency transition to %u kHz\n",
 331                 state, freqs->new);
 332
 333         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 334         switch (state) {
 335
 336         case CPUFREQ_PRECHANGE:
 337                 /* detect if the driver reported a value as "old frequency"
 338                  * which is not equal to what the cpufreq core thinks is
 339                  * "old frequency".
 340                  */
 341                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 342                         if ((policy) && (policy->cpu == freqs->cpu) &&
 343                             (policy->cur) && (policy->cur != freqs->old)) {
 344                                 dprintk("Warning: CPU frequency is"
 345                                         " %u, cpufreq assumed %u kHz.\n",
 346                                         freqs->old, policy->cur);
 347                                 freqs->old = policy->cur;
 348                         }
 349                 }
 350                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 351                                 CPUFREQ_PRECHANGE, freqs);
 352                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 353                 break;
 354
 355         case CPUFREQ_POSTCHANGE:
 356                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 357                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 358                                 CPUFREQ_POSTCHANGE, freqs);
 359                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 360                         policy->cur = freqs->new;
 361                 break;
 362         }
 363 }
 364 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 365
 366
 367
 368 /*********************************************************************
 369  *                          SYSFS INTERFACE                          *
 370  *********************************************************************/
 371
 372 static struct cpufreq_governor *__find_governor(const char *str_governor)
 373 {
 374         struct cpufreq_governor *t;
 375
 376         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 377                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 378                         return t;
 379
 380         return NULL;
 381 }
 382
 383 /**
 384  * cpufreq_parse_governor - parse a governor string
 385  */
 386 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 387                                 struct cpufreq_governor **governor)
 388 {
 389         int err = -EINVAL;
 390
 391         if (!cpufreq_driver)
 392                 goto out;
 393
 394         if (cpufreq_driver->setpolicy) {
 395                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 396                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 397                         err = 0;
 398                 } else if (!strnicmp(str_governor, "powersave",
 399                                                 CPUFREQ_NAME_LEN)) {
 400                         *policy = CPUFREQ_POLICY_POWERSAVE;
 401                         err = 0;
 402                 }
 403         } else if (cpufreq_driver->target) {
 404                 struct cpufreq_governor *t;
 405
 406                 mutex_lock(&cpufreq_governor_mutex);
 407
 408                 t = __find_governor(str_governor);
 409
 410                 if (t == NULL) {
 411                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 412                                                                 str_governor);
 413
 414                         if (name) {
 415                                 int ret;
 416
 417                                 mutex_unlock(&cpufreq_governor_mutex);
 418                                 ret = request_module("%s", name);
 419                                 mutex_lock(&cpufreq_governor_mutex);
 420
 421                                 if (ret == 0)
 422                                         t = __find_governor(str_governor);
 423                         }
 424
 425                         kfree(name);
 426                 }
 427
 428                 if (t != NULL) {
 429                         *governor = t;
 430                         err = 0;
 431                 }
 432
 433                 mutex_unlock(&cpufreq_governor_mutex);
 434         }
 435 out:
 436         return err;
 437 }
 438
 439
 440 /**
 441  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 442  * print out cpufreq information
 443  *
 444  * Write out information from cpufreq_driver->policy[cpu]; object must be
 445  * "unsigned int".
 446  */
 447
 448 #define show_one(file_name, object)                     \
 449 static ssize_t show_##file_name                         \
 450 (struct cpufreq_policy *policy, char *buf)              \
 451 {                                                       \
 452         return sprintf(buf, "%u\n", policy->object);    \
 453 }
 454
 455 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 456 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 457 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret = -EINVAL;                                     \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf(buf, "%u", &new_policy.object);                    \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq, min);
 490 store_one(scaling_max_freq, max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504
 505 /**
 506  * show_scaling_governor - show the current policy for the specified CPU
 507  */
 508 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 509 {
 510         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 511                 return sprintf(buf, "powersave\n");
 512         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 513                 return sprintf(buf, "performance\n");
 514         else if (policy->governor)
 515                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 516                                 policy->governor->name);
 517         return -EINVAL;
 518 }
 519
 520
 521 /**
 522  * store_scaling_governor - store policy for the specified CPU
 523  */
 524 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 525                                         const char *buf, size_t count)
 526 {
 527         unsigned int ret = -EINVAL;
 528         char    str_governor[16];
 529         struct cpufreq_policy new_policy;
 530
 531         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 532         if (ret)
 533                 return ret;
 534
 535         ret = sscanf(buf, "%15s", str_governor);
 536         if (ret != 1)
 537                 return -EINVAL;
 538
 539         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 540                                                 &new_policy.governor))
 541                 return -EINVAL;
 542
 543         /* Do not use cpufreq_set_policy here or the user_policy.max
 544            will be wrongly overridden */
 545         ret = __cpufreq_set_policy(policy, &new_policy);
 546
 547         policy->user_policy.policy = policy->policy;
 548         policy->user_policy.governor = policy->governor;
 549
 550         if (ret)
 551                 return ret;
 552         else
 553                 return count;
 554 }
 555
 556 /**
 557  * show_scaling_driver - show the cpufreq driver currently loaded
 558  */
 559 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 560 {
 561         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 562 }
 563
 564 /**
 565  * show_scaling_available_governors - show the available CPUfreq governors
 566  */
 567 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 568                                                 char *buf)
 569 {
 570         ssize_t i = 0;
 571         struct cpufreq_governor *t;
 572
 573         if (!cpufreq_driver->target) {
 574                 i += sprintf(buf, "performance powersave");
 575                 goto out;
 576         }
 577
 578         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 579                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 580                     - (CPUFREQ_NAME_LEN + 2)))
 581                         goto out;
 582                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 583         }
 584 out:
 585         i += sprintf(&buf[i], "\n");
 586         return i;
 587 }
 588
 589 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu(cpu, mask) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                         break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604
 605 /**
 606  * show_related_cpus - show the CPUs affected by each transition even if
 607  * hw coordination is in use
 608  */
 609 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 610 {
 611         if (cpumask_empty(policy->related_cpus))
 612                 return show_cpus(policy->cpus, buf);
 613         return show_cpus(policy->related_cpus, buf);
 614 }
 615
 616 /**
 617  * show_affected_cpus - show the CPUs affected by each transition
 618  */
 619 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 620 {
 621         return show_cpus(policy->cpus, buf);
 622 }
 623
 624 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 625                                         const char *buf, size_t count)
 626 {
 627         unsigned int freq = 0;
 628         unsigned int ret;
 629
 630         if (!policy->governor || !policy->governor->store_setspeed)
 631                 return -EINVAL;
 632
 633         ret = sscanf(buf, "%u", &freq);
 634         if (ret != 1)
 635                 return -EINVAL;
 636
 637         policy->governor->store_setspeed(policy, freq);
 638
 639         return count;
 640 }
 641
 642 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 643 {
 644         if (!policy->governor || !policy->governor->show_setspeed)
 645                 return sprintf(buf, "<unsupported>\n");
 646
 647         return policy->governor->show_setspeed(policy, buf);
 648 }
 649
 650 #define define_one_ro(_name) \
 651 static struct freq_attr _name = \
 652 __ATTR(_name, 0444, show_##_name, NULL)
 653
 654 #define define_one_ro0400(_name) \
 655 static struct freq_attr _name = \
 656 __ATTR(_name, 0400, show_##_name, NULL)
 657
 658 #define define_one_rw(_name) \
 659 static struct freq_attr _name = \
 660 __ATTR(_name, 0644, show_##_name, store_##_name)
 661
 662 define_one_ro0400(cpuinfo_cur_freq);
 663 define_one_ro(cpuinfo_min_freq);
 664 define_one_ro(cpuinfo_max_freq);
 665 define_one_ro(cpuinfo_transition_latency);
 666 define_one_ro(scaling_available_governors);
 667 define_one_ro(scaling_driver);
 668 define_one_ro(scaling_cur_freq);
 669 define_one_ro(related_cpus);
 670 define_one_ro(affected_cpus);
 671 define_one_rw(scaling_min_freq);
 672 define_one_rw(scaling_max_freq);
 673 define_one_rw(scaling_governor);
 674 define_one_rw(scaling_setspeed);
 675
 676 static struct attribute *default_attrs[] = {
 677         &cpuinfo_min_freq.attr,
 678         &cpuinfo_max_freq.attr,
 679         &cpuinfo_transition_latency.attr,
 680         &scaling_min_freq.attr,
 681         &scaling_max_freq.attr,
 682         &affected_cpus.attr,
 683         &related_cpus.attr,
 684         &scaling_governor.attr,
 685         &scaling_driver.attr,
 686         &scaling_available_governors.attr,
 687         &scaling_setspeed.attr,
 688         NULL
 689 };
 690
 691 struct kobject *cpufreq_global_kobject;
 692 EXPORT_SYMBOL(cpufreq_global_kobject);
 693
 694 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 695 #define to_attr(a) container_of(a, struct freq_attr, attr)
 696
 697 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         struct cpufreq_policy *policy = to_policy(kobj);
 700         struct freq_attr *fattr = to_attr(attr);
 701         ssize_t ret = -EINVAL;
 702         policy = cpufreq_cpu_get(policy->cpu);
 703         if (!policy)
 704                 goto no_policy;
 705
 706         if (lock_policy_rwsem_read(policy->cpu) < 0)
 707                 goto fail;
 708
 709         if (fattr->show)
 710                 ret = fattr->show(policy, buf);
 711         else
 712                 ret = -EIO;
 713
 714         unlock_policy_rwsem_read(policy->cpu);
 715 fail:
 716         cpufreq_cpu_put(policy);
 717 no_policy:
 718         return ret;
 719 }
 720
 721 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 722                      const char *buf, size_t count)
 723 {
 724         struct cpufreq_policy *policy = to_policy(kobj);
 725         struct freq_attr *fattr = to_attr(attr);
 726         ssize_t ret = -EINVAL;
 727         policy = cpufreq_cpu_get(policy->cpu);
 728         if (!policy)
 729                 goto no_policy;
 730
 731         if (lock_policy_rwsem_write(policy->cpu) < 0)
 732                 goto fail;
 733
 734         if (fattr->store)
 735                 ret = fattr->store(policy, buf, count);
 736         else
 737                 ret = -EIO;
 738
 739         unlock_policy_rwsem_write(policy->cpu);
 740 fail:
 741         cpufreq_cpu_put(policy);
 742 no_policy:
 743         return ret;
 744 }
 745
 746 static void cpufreq_sysfs_release(struct kobject *kobj)
 747 {
 748         struct cpufreq_policy *policy = to_policy(kobj);
 749         dprintk("last reference is dropped\n");
 750         complete(&policy->kobj_unregister);
 751 }
 752
 753 static struct sysfs_ops sysfs_ops = {
 754         .show   = show,
 755         .store  = store,
 756 };
 757
 758 static struct kobj_type ktype_cpufreq = {
 759         .sysfs_ops      = &sysfs_ops,
 760         .default_attrs  = default_attrs,
 761         .release        = cpufreq_sysfs_release,
 762 };
 763
 764 /*
 765  * Returns:
 766  *   Negative: Failure
 767  *   0:        Success
 768  *   Positive: When we have a managed CPU and the sysfs got symlinked
 769  */
 770 int cpufreq_add_dev_policy(unsigned int cpu, struct cpufreq_policy *policy,
 771                 struct sys_device *sys_dev)
 772 {
 773         int ret = 0;
 774 #ifdef CONFIG_SMP
 775         unsigned long flags;
 776         unsigned int j;
 777 #ifdef CONFIG_HOTPLUG_CPU
 778         struct cpufreq_governor *gov;
 779
 780         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
 781         if (gov) {
 782                 policy->governor = gov;
 783                 dprintk("Restoring governor %s for cpu %d\n",
 784                        policy->governor->name, cpu);
 785         }
 786 #endif
 787
 788         for_each_cpu(j, policy->cpus) {
 789                 struct cpufreq_policy *managed_policy;
 790
 791                 if (cpu == j)
 792                         continue;
 793
 794                 /* Check for existing affected CPUs.
 795                  * They may not be aware of it due to CPU Hotplug.
 796                  * cpufreq_cpu_put is called when the device is removed
 797                  * in __cpufreq_remove_dev()
 798                  */
 799                 managed_policy = cpufreq_cpu_get(j);
 800                 if (unlikely(managed_policy)) {
 801
 802                         /* Set proper policy_cpu */
 803                         unlock_policy_rwsem_write(cpu);
 804                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 805
 806                         if (lock_policy_rwsem_write(cpu) < 0) {
 807                                 /* Should not go through policy unlock path */
 808                                 if (cpufreq_driver->exit)
 809                                         cpufreq_driver->exit(policy);
 810                                 cpufreq_cpu_put(managed_policy);
 811                                 return -EBUSY;
 812                         }
 813
 814                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 815                         cpumask_copy(managed_policy->cpus, policy->cpus);
 816                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 817                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 818
 819                         dprintk("CPU already managed, adding link\n");
 820                         ret = sysfs_create_link(&sys_dev->kobj,
 821                                                 &managed_policy->kobj,
 822                                                 "cpufreq");
 823                         if (ret)
 824                                 cpufreq_cpu_put(managed_policy);
 825                         /*
 826                          * Success. We only needed to be added to the mask.
 827                          * Call driver->exit() because only the cpu parent of
 828                          * the kobj needed to call init().
 829                          */
 830                         if (cpufreq_driver->exit)
 831                                 cpufreq_driver->exit(policy);
 832
 833                         if (!ret)
 834                                 return 1;
 835                         else
 836                                 return ret;
 837                 }
 838         }
 839 #endif
 840         return ret;
 841 }
 842
 843
 844 /* symlink affected CPUs */
 845 int cpufreq_add_dev_symlink(unsigned int cpu, struct cpufreq_policy *policy)
 846 {
 847         unsigned int j;
 848         int ret = 0;
 849
 850         for_each_cpu(j, policy->cpus) {
 851                 struct cpufreq_policy *managed_policy;
 852                 struct sys_device *cpu_sys_dev;
 853
 854                 if (j == cpu)
 855                         continue;
 856                 if (!cpu_online(j))
 857                         continue;
 858
 859                 dprintk("CPU %u already managed, adding link\n", j);
 860                 managed_policy = cpufreq_cpu_get(cpu);
 861                 cpu_sys_dev = get_cpu_sysdev(j);
 862                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 863                                         "cpufreq");
 864                 if (ret) {
 865                         cpufreq_cpu_put(managed_policy);
 866                         return ret;
 867                 }
 868         }
 869         return ret;
 870 }
 871
 872 int cpufreq_add_dev_interface(unsigned int cpu, struct cpufreq_policy *policy,
 873                 struct sys_device *sys_dev)
 874 {
 875         struct cpufreq_policy new_policy;
 876         struct freq_attr **drv_attr;
 877         unsigned long flags;
 878         int ret = 0;
 879         unsigned int j;
 880
 881         /* prepare interface data */
 882         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 883                                    &sys_dev->kobj, "cpufreq");
 884         if (ret)
 885                 return ret;
 886
 887         /* set up files for this cpu device */
 888         drv_attr = cpufreq_driver->attr;
 889         while ((drv_attr) && (*drv_attr)) {
 890                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 891                 if (ret)
 892                         goto err_out_kobj_put;
 893                 drv_attr++;
 894         }
 895         if (cpufreq_driver->get) {
 896                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 897                 if (ret)
 898                         goto err_out_kobj_put;
 899         }
 900         if (cpufreq_driver->target) {
 901                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 902                 if (ret)
 903                         goto err_out_kobj_put;
 904         }
 905
 906         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 907         for_each_cpu(j, policy->cpus) {
 908         if (!cpu_online(j))
 909                 continue;
 910                 per_cpu(cpufreq_cpu_data, j) = policy;
 911                 per_cpu(policy_cpu, j) = policy->cpu;
 912         }
 913         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 914
 915         ret = cpufreq_add_dev_symlink(cpu, policy);
 916         if (ret)
 917                 goto err_out_kobj_put;
 918
 919         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 920         /* assure that the starting sequence is run in __cpufreq_set_policy */
 921         policy->governor = NULL;
 922
 923         /* set default policy */
 924         ret = __cpufreq_set_policy(policy, &new_policy);
 925         policy->user_policy.policy = policy->policy;
 926         policy->user_policy.governor = policy->governor;
 927
 928         if (ret) {
 929                 dprintk("setting policy failed\n");
 930                 if (cpufreq_driver->exit)
 931                         cpufreq_driver->exit(policy);
 932         }
 933         return ret;
 934
 935 err_out_kobj_put:
 936         kobject_put(&policy->kobj);
 937         wait_for_completion(&policy->kobj_unregister);
 938         return ret;
 939 }
 940
 941
 942 /**
 943  * cpufreq_add_dev - add a CPU device
 944  *
 945  * Adds the cpufreq interface for a CPU device.
 946  *
 947  * The Oracle says: try running cpufreq registration/unregistration concurrently
 948  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 949  * mess up, but more thorough testing is needed. - Mathieu
 950  */
 951 static int cpufreq_add_dev(struct sys_device *sys_dev)
 952 {
 953         unsigned int cpu = sys_dev->id;
 954         int ret = 0;
 955         struct cpufreq_policy *policy;
 956         unsigned long flags;
 957         unsigned int j;
 958
 959         if (cpu_is_offline(cpu))
 960                 return 0;
 961
 962         cpufreq_debug_disable_ratelimit();
 963         dprintk("adding CPU %u\n", cpu);
 964
 965 #ifdef CONFIG_SMP
 966         /* check whether a different CPU already registered this
 967          * CPU because it is in the same boat. */
 968         policy = cpufreq_cpu_get(cpu);
 969         if (unlikely(policy)) {
 970                 cpufreq_cpu_put(policy);
 971                 cpufreq_debug_enable_ratelimit();
 972                 return 0;
 973         }
 974 #endif
 975
 976         if (!try_module_get(cpufreq_driver->owner)) {
 977                 ret = -EINVAL;
 978                 goto module_out;
 979         }
 980
 981         ret = -ENOMEM;
 982         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 983         if (!policy)
 984                 goto nomem_out;
 985
 986         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 987                 goto err_free_policy;
 988
 989         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 990                 goto err_free_cpumask;
 991
 992         policy->cpu = cpu;
 993         cpumask_copy(policy->cpus, cpumask_of(cpu));
 994
 995         /* Initially set CPU itself as the policy_cpu */
 996         per_cpu(policy_cpu, cpu) = cpu;
 997         ret = (lock_policy_rwsem_write(cpu) < 0);
 998         WARN_ON(ret);
 999
1000         init_completion(&policy->kobj_unregister);
1001         INIT_WORK(&policy->update, handle_update);
1002
1003         /* Set governor before ->init, so that driver could check it */
1004         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1005         /* call driver. From then on the cpufreq must be able
1006          * to accept all calls to ->verify and ->setpolicy for this CPU
1007          */
1008         ret = cpufreq_driver->init(policy);
1009         if (ret) {
1010                 dprintk("initialization failed\n");
1011                 goto err_unlock_policy;
1012         }
1013         policy->user_policy.min = policy->min;
1014         policy->user_policy.max = policy->max;
1015
1016         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1017                                      CPUFREQ_START, policy);
1018
1019         ret = cpufreq_add_dev_policy(cpu, policy, sys_dev);
1020         if (ret) {
1021                 if (ret > 0)
1022                         /* This is a managed cpu, symlink created,
1023                            exit with 0 */
1024                         ret = 0;
1025                 goto err_unlock_policy;
1026         }
1027
1028         ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
1029         if (ret)
1030                 goto err_out_unregister;
1031
1032         unlock_policy_rwsem_write(cpu);
1033
1034         kobject_uevent(&policy->kobj, KOBJ_ADD);
1035         module_put(cpufreq_driver->owner);
1036         dprintk("initialization complete\n");
1037         cpufreq_debug_enable_ratelimit();
1038
1039         return 0;
1040
1041
1042 err_out_unregister:
1043         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1044         for_each_cpu(j, policy->cpus)
1045                 per_cpu(cpufreq_cpu_data, j) = NULL;
1046         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1047
1048         kobject_put(&policy->kobj);
1049         wait_for_completion(&policy->kobj_unregister);
1050
1051 err_unlock_policy:
1052         unlock_policy_rwsem_write(cpu);
1053 err_free_cpumask:
1054         free_cpumask_var(policy->cpus);
1055 err_free_policy:
1056         kfree(policy);
1057 nomem_out:
1058         module_put(cpufreq_driver->owner);
1059 module_out:
1060         cpufreq_debug_enable_ratelimit();
1061         return ret;
1062 }
1063
1064
1065 /**
1066  * __cpufreq_remove_dev - remove a CPU device
1067  *
1068  * Removes the cpufreq interface for a CPU device.
1069  * Caller should already have policy_rwsem in write mode for this CPU.
1070  * This routine frees the rwsem before returning.
1071  */
1072 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1073 {
1074         unsigned int cpu = sys_dev->id;
1075         unsigned long flags;
1076         struct cpufreq_policy *data;
1077 #ifdef CONFIG_SMP
1078         struct sys_device *cpu_sys_dev;
1079         unsigned int j;
1080 #endif
1081
1082         cpufreq_debug_disable_ratelimit();
1083         dprintk("unregistering CPU %u\n", cpu);
1084
1085         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1086         data = per_cpu(cpufreq_cpu_data, cpu);
1087
1088         if (!data) {
1089                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1090                 cpufreq_debug_enable_ratelimit();
1091                 unlock_policy_rwsem_write(cpu);
1092                 return -EINVAL;
1093         }
1094         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1095
1096
1097 #ifdef CONFIG_SMP
1098         /* if this isn't the CPU which is the parent of the kobj, we
1099          * only need to unlink, put and exit
1100          */
1101         if (unlikely(cpu != data->cpu)) {
1102                 dprintk("removing link\n");
1103                 cpumask_clear_cpu(cpu, data->cpus);
1104                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1105                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1106                 cpufreq_cpu_put(data);
1107                 cpufreq_debug_enable_ratelimit();
1108                 unlock_policy_rwsem_write(cpu);
1109                 return 0;
1110         }
1111 #endif
1112
1113 #ifdef CONFIG_SMP
1114
1115 #ifdef CONFIG_HOTPLUG_CPU
1116         strncpy(per_cpu(cpufreq_cpu_governor, cpu), data->governor->name,
1117                         CPUFREQ_NAME_LEN);
1118 #endif
1119
1120         /* if we have other CPUs still registered, we need to unlink them,
1121          * or else wait_for_completion below will lock up. Clean the
1122          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1123          * the sysfs links afterwards.
1124          */
1125         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1126                 for_each_cpu(j, data->cpus) {
1127                         if (j == cpu)
1128                                 continue;
1129                         per_cpu(cpufreq_cpu_data, j) = NULL;
1130                 }
1131         }
1132
1133         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1134
1135         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1136                 for_each_cpu(j, data->cpus) {
1137                         if (j == cpu)
1138                                 continue;
1139                         dprintk("removing link for cpu %u\n", j);
1140 #ifdef CONFIG_HOTPLUG_CPU
1141                         strncpy(per_cpu(cpufreq_cpu_governor, j),
1142                                 data->governor->name, CPUFREQ_NAME_LEN);
1143 #endif
1144                         cpu_sys_dev = get_cpu_sysdev(j);
1145                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1146                         cpufreq_cpu_put(data);
1147                 }
1148         }
1149 #else
1150         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1151 #endif
1152
1153         if (cpufreq_driver->target)
1154                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1155
1156         kobject_put(&data->kobj);
1157
1158         /* we need to make sure that the underlying kobj is actually
1159          * not referenced anymore by anybody before we proceed with
1160          * unloading.
1161          */
1162         dprintk("waiting for dropping of refcount\n");
1163         wait_for_completion(&data->kobj_unregister);
1164         dprintk("wait complete\n");
1165
1166         if (cpufreq_driver->exit)
1167                 cpufreq_driver->exit(data);
1168
1169         unlock_policy_rwsem_write(cpu);
1170
1171         free_cpumask_var(data->related_cpus);
1172         free_cpumask_var(data->cpus);
1173         kfree(data);
1174         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1175
1176         cpufreq_debug_enable_ratelimit();
1177         return 0;
1178 }
1179
1180
1181 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1182 {
1183         unsigned int cpu = sys_dev->id;
1184         int retval;
1185
1186         if (cpu_is_offline(cpu))
1187                 return 0;
1188
1189         if (unlikely(lock_policy_rwsem_write(cpu)))
1190                 BUG();
1191
1192         retval = __cpufreq_remove_dev(sys_dev);
1193         return retval;
1194 }
1195
1196
1197 static void handle_update(struct work_struct *work)
1198 {
1199         struct cpufreq_policy *policy =
1200                 container_of(work, struct cpufreq_policy, update);
1201         unsigned int cpu = policy->cpu;
1202         dprintk("handle_update for cpu %u called\n", cpu);
1203         cpufreq_update_policy(cpu);
1204 }
1205
1206 /**
1207  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1208  *      @cpu: cpu number
1209  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1210  *      @new_freq: CPU frequency the CPU actually runs at
1211  *
1212  *      We adjust to current frequency first, and need to clean up later.
1213  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1214  */
1215 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1216                                 unsigned int new_freq)
1217 {
1218         struct cpufreq_freqs freqs;
1219
1220         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1221                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1222
1223         freqs.cpu = cpu;
1224         freqs.old = old_freq;
1225         freqs.new = new_freq;
1226         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1227         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1228 }
1229
1230
1231 /**
1232  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1233  * @cpu: CPU number
1234  *
1235  * This is the last known freq, without actually getting it from the driver.
1236  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1237  */
1238 unsigned int cpufreq_quick_get(unsigned int cpu)
1239 {
1240         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1241         unsigned int ret_freq = 0;
1242
1243         if (policy) {
1244                 ret_freq = policy->cur;
1245                 cpufreq_cpu_put(policy);
1246         }
1247
1248         return ret_freq;
1249 }
1250 EXPORT_SYMBOL(cpufreq_quick_get);
1251
1252
1253 static unsigned int __cpufreq_get(unsigned int cpu)
1254 {
1255         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1256         unsigned int ret_freq = 0;
1257
1258         if (!cpufreq_driver->get)
1259                 return ret_freq;
1260
1261         ret_freq = cpufreq_driver->get(cpu);
1262
1263         if (ret_freq && policy->cur &&
1264                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1265                 /* verify no discrepancy between actual and
1266                                         saved value exists */
1267                 if (unlikely(ret_freq != policy->cur)) {
1268                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1269                         schedule_work(&policy->update);
1270                 }
1271         }
1272
1273         return ret_freq;
1274 }
1275
1276 /**
1277  * cpufreq_get - get the current CPU frequency (in kHz)
1278  * @cpu: CPU number
1279  *
1280  * Get the CPU current (static) CPU frequency
1281  */
1282 unsigned int cpufreq_get(unsigned int cpu)
1283 {
1284         unsigned int ret_freq = 0;
1285         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1286
1287         if (!policy)
1288                 goto out;
1289
1290         if (unlikely(lock_policy_rwsem_read(cpu)))
1291                 goto out_policy;
1292
1293         ret_freq = __cpufreq_get(cpu);
1294
1295         unlock_policy_rwsem_read(cpu);
1296
1297 out_policy:
1298         cpufreq_cpu_put(policy);
1299 out:
1300         return ret_freq;
1301 }
1302 EXPORT_SYMBOL(cpufreq_get);
1303
1304
1305 /**
1306  *      cpufreq_suspend - let the low level driver prepare for suspend
1307  */
1308
1309 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1310 {
1311         int ret = 0;
1312
1313         int cpu = sysdev->id;
1314         struct cpufreq_policy *cpu_policy;
1315
1316         dprintk("suspending cpu %u\n", cpu);
1317
1318         if (!cpu_online(cpu))
1319                 return 0;
1320
1321         /* we may be lax here as interrupts are off. Nonetheless
1322          * we need to grab the correct cpu policy, as to check
1323          * whether we really run on this CPU.
1324          */
1325
1326         cpu_policy = cpufreq_cpu_get(cpu);
1327         if (!cpu_policy)
1328                 return -EINVAL;
1329
1330         /* only handle each CPU group once */
1331         if (unlikely(cpu_policy->cpu != cpu))
1332                 goto out;
1333
1334         if (cpufreq_driver->suspend) {
1335                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1336                 if (ret)
1337                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1338                                         "step on CPU %u\n", cpu_policy->cpu);
1339         }
1340
1341 out:
1342         cpufreq_cpu_put(cpu_policy);
1343         return ret;
1344 }
1345
1346 /**
1347  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1348  *
1349  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1350  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1351  *          restored. It will verify that the current freq is in sync with
1352  *          what we believe it to be. This is a bit later than when it
1353  *          should be, but nonethteless it's better than calling
1354  *          cpufreq_driver->get() here which might re-enable interrupts...
1355  */
1356 static int cpufreq_resume(struct sys_device *sysdev)
1357 {
1358         int ret = 0;
1359
1360         int cpu = sysdev->id;
1361         struct cpufreq_policy *cpu_policy;
1362
1363         dprintk("resuming cpu %u\n", cpu);
1364
1365         if (!cpu_online(cpu))
1366                 return 0;
1367
1368         /* we may be lax here as interrupts are off. Nonetheless
1369          * we need to grab the correct cpu policy, as to check
1370          * whether we really run on this CPU.
1371          */
1372
1373         cpu_policy = cpufreq_cpu_get(cpu);
1374         if (!cpu_policy)
1375                 return -EINVAL;
1376
1377         /* only handle each CPU group once */
1378         if (unlikely(cpu_policy->cpu != cpu))
1379                 goto fail;
1380
1381         if (cpufreq_driver->resume) {
1382                 ret = cpufreq_driver->resume(cpu_policy);
1383                 if (ret) {
1384                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1385                                         "step on CPU %u\n", cpu_policy->cpu);
1386                         goto fail;
1387                 }
1388         }
1389
1390         schedule_work(&cpu_policy->update);
1391
1392 fail:
1393         cpufreq_cpu_put(cpu_policy);
1394         return ret;
1395 }
1396
1397 static struct sysdev_driver cpufreq_sysdev_driver = {
1398         .add            = cpufreq_add_dev,
1399         .remove         = cpufreq_remove_dev,
1400         .suspend        = cpufreq_suspend,
1401         .resume         = cpufreq_resume,
1402 };
1403
1404
1405 /*********************************************************************
1406  *                     NOTIFIER LISTS INTERFACE                      *
1407  *********************************************************************/
1408
1409 /**
1410  *      cpufreq_register_notifier - register a driver with cpufreq
1411  *      @nb: notifier function to register
1412  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1413  *
1414  *      Add a driver to one of two lists: either a list of drivers that
1415  *      are notified about clock rate changes (once before and once after
1416  *      the transition), or a list of drivers that are notified about
1417  *      changes in cpufreq policy.
1418  *
1419  *      This function may sleep, and has the same return conditions as
1420  *      blocking_notifier_chain_register.
1421  */
1422 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1423 {
1424         int ret;
1425
1426         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1427
1428         switch (list) {
1429         case CPUFREQ_TRANSITION_NOTIFIER:
1430                 ret = srcu_notifier_chain_register(
1431                                 &cpufreq_transition_notifier_list, nb);
1432                 break;
1433         case CPUFREQ_POLICY_NOTIFIER:
1434                 ret = blocking_notifier_chain_register(
1435                                 &cpufreq_policy_notifier_list, nb);
1436                 break;
1437         default:
1438                 ret = -EINVAL;
1439         }
1440
1441         return ret;
1442 }
1443 EXPORT_SYMBOL(cpufreq_register_notifier);
1444
1445
1446 /**
1447  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1448  *      @nb: notifier block to be unregistered
1449  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1450  *
1451  *      Remove a driver from the CPU frequency notifier list.
1452  *
1453  *      This function may sleep, and has the same return conditions as
1454  *      blocking_notifier_chain_unregister.
1455  */
1456 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1457 {
1458         int ret;
1459
1460         switch (list) {
1461         case CPUFREQ_TRANSITION_NOTIFIER:
1462                 ret = srcu_notifier_chain_unregister(
1463                                 &cpufreq_transition_notifier_list, nb);
1464                 break;
1465         case CPUFREQ_POLICY_NOTIFIER:
1466                 ret = blocking_notifier_chain_unregister(
1467                                 &cpufreq_policy_notifier_list, nb);
1468                 break;
1469         default:
1470                 ret = -EINVAL;
1471         }
1472
1473         return ret;
1474 }
1475 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1476
1477
1478 /*********************************************************************
1479  *                              GOVERNORS                            *
1480  *********************************************************************/
1481
1482
1483 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1484                             unsigned int target_freq,
1485                             unsigned int relation)
1486 {
1487         int retval = -EINVAL;
1488
1489         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1490                 target_freq, relation);
1491         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1492                 retval = cpufreq_driver->target(policy, target_freq, relation);
1493
1494         return retval;
1495 }
1496 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1497
1498 int cpufreq_driver_target(struct cpufreq_policy *policy,
1499                           unsigned int target_freq,
1500                           unsigned int relation)
1501 {
1502         int ret = -EINVAL;
1503
1504         policy = cpufreq_cpu_get(policy->cpu);
1505         if (!policy)
1506                 goto no_policy;
1507
1508         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1509                 goto fail;
1510
1511         ret = __cpufreq_driver_target(policy, target_freq, relation);
1512
1513         unlock_policy_rwsem_write(policy->cpu);
1514
1515 fail:
1516         cpufreq_cpu_put(policy);
1517 no_policy:
1518         return ret;
1519 }
1520 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1521
1522 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1523 {
1524         int ret = 0;
1525
1526         policy = cpufreq_cpu_get(policy->cpu);
1527         if (!policy)
1528                 return -EINVAL;
1529
1530         if (cpu_online(cpu) && cpufreq_driver->getavg)
1531                 ret = cpufreq_driver->getavg(policy, cpu);
1532
1533         cpufreq_cpu_put(policy);
1534         return ret;
1535 }
1536 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1537
1538 /*
1539  * when "event" is CPUFREQ_GOV_LIMITS
1540  */
1541
1542 static int __cpufreq_governor(struct cpufreq_policy *policy,
1543                                         unsigned int event)
1544 {
1545         int ret;
1546
1547         /* Only must be defined when default governor is known to have latency
1548            restrictions, like e.g. conservative or ondemand.
1549            That this is the case is already ensured in Kconfig
1550         */
1551 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1552         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1553 #else
1554         struct cpufreq_governor *gov = NULL;
1555 #endif
1556
1557         if (policy->governor->max_transition_latency &&
1558             policy->cpuinfo.transition_latency >
1559             policy->governor->max_transition_latency) {
1560                 if (!gov)
1561                         return -EINVAL;
1562                 else {
1563                         printk(KERN_WARNING "%s governor failed, too long"
1564                                " transition latency of HW, fallback"
1565                                " to %s governor\n",
1566                                policy->governor->name,
1567                                gov->name);
1568                         policy->governor = gov;
1569                 }
1570         }
1571
1572         if (!try_module_get(policy->governor->owner))
1573                 return -EINVAL;
1574
1575         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1576                                                 policy->cpu, event);
1577         ret = policy->governor->governor(policy, event);
1578
1579         /* we keep one module reference alive for
1580                         each CPU governed by this CPU */
1581         if ((event != CPUFREQ_GOV_START) || ret)
1582                 module_put(policy->governor->owner);
1583         if ((event == CPUFREQ_GOV_STOP) && !ret)
1584                 module_put(policy->governor->owner);
1585
1586         return ret;
1587 }
1588
1589
1590 int cpufreq_register_governor(struct cpufreq_governor *governor)
1591 {
1592         int err;
1593
1594         if (!governor)
1595                 return -EINVAL;
1596
1597         mutex_lock(&cpufreq_governor_mutex);
1598
1599         err = -EBUSY;
1600         if (__find_governor(governor->name) == NULL) {
1601                 err = 0;
1602                 list_add(&governor->governor_list, &cpufreq_governor_list);
1603         }
1604
1605         mutex_unlock(&cpufreq_governor_mutex);
1606         return err;
1607 }
1608 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1609
1610
1611 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1612 {
1613         if (!governor)
1614                 return;
1615
1616         mutex_lock(&cpufreq_governor_mutex);
1617         list_del(&governor->governor_list);
1618         mutex_unlock(&cpufreq_governor_mutex);
1619         return;
1620 }
1621 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1622
1623
1624
1625 /*********************************************************************
1626  *                          POLICY INTERFACE                         *
1627  *********************************************************************/
1628
1629 /**
1630  * cpufreq_get_policy - get the current cpufreq_policy
1631  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1632  *      is written
1633  *
1634  * Reads the current cpufreq policy.
1635  */
1636 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1637 {
1638         struct cpufreq_policy *cpu_policy;
1639         if (!policy)
1640                 return -EINVAL;
1641
1642         cpu_policy = cpufreq_cpu_get(cpu);
1643         if (!cpu_policy)
1644                 return -EINVAL;
1645
1646         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1647
1648         cpufreq_cpu_put(cpu_policy);
1649         return 0;
1650 }
1651 EXPORT_SYMBOL(cpufreq_get_policy);
1652
1653
1654 /*
1655  * data   : current policy.
1656  * policy : policy to be set.
1657  */
1658 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1659                                 struct cpufreq_policy *policy)
1660 {
1661         int ret = 0;
1662
1663         cpufreq_debug_disable_ratelimit();
1664         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1665                 policy->min, policy->max);
1666
1667         memcpy(&policy->cpuinfo, &data->cpuinfo,
1668                                 sizeof(struct cpufreq_cpuinfo));
1669
1670         if (policy->min > data->max || policy->max < data->min) {
1671                 ret = -EINVAL;
1672                 goto error_out;
1673         }
1674
1675         /* verify the cpu speed can be set within this limit */
1676         ret = cpufreq_driver->verify(policy);
1677         if (ret)
1678                 goto error_out;
1679
1680         /* adjust if necessary - all reasons */
1681         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1682                         CPUFREQ_ADJUST, policy);
1683
1684         /* adjust if necessary - hardware incompatibility*/
1685         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1686                         CPUFREQ_INCOMPATIBLE, policy);
1687
1688         /* verify the cpu speed can be set within this limit,
1689            which might be different to the first one */
1690         ret = cpufreq_driver->verify(policy);
1691         if (ret)
1692                 goto error_out;
1693
1694         /* notification of the new policy */
1695         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1696                         CPUFREQ_NOTIFY, policy);
1697
1698         data->min = policy->min;
1699         data->max = policy->max;
1700
1701         dprintk("new min and max freqs are %u - %u kHz\n",
1702                                         data->min, data->max);
1703
1704         if (cpufreq_driver->setpolicy) {
1705                 data->policy = policy->policy;
1706                 dprintk("setting range\n");
1707                 ret = cpufreq_driver->setpolicy(policy);
1708         } else {
1709                 if (policy->governor != data->governor) {
1710                         /* save old, working values */
1711                         struct cpufreq_governor *old_gov = data->governor;
1712
1713                         dprintk("governor switch\n");
1714
1715                         /* end old governor */
1716                         if (data->governor) {
1717                                 /*
1718                                  * Need to release the rwsem around governor
1719                                  * stop due to lock dependency between
1720                                  * cancel_delayed_work_sync and the read lock
1721                                  * taken in the delayed work handler.
1722                                  */
1723                                 unlock_policy_rwsem_write(data->cpu);
1724                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1725                                 lock_policy_rwsem_write(data->cpu);
1726                         }
1727
1728                         /* start new governor */
1729                         data->governor = policy->governor;
1730                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1731                                 /* new governor failed, so re-start old one */
1732                                 dprintk("starting governor %s failed\n",
1733                                                         data->governor->name);
1734                                 if (old_gov) {
1735                                         data->governor = old_gov;
1736                                         __cpufreq_governor(data,
1737                                                            CPUFREQ_GOV_START);
1738                                 }
1739                                 ret = -EINVAL;
1740                                 goto error_out;
1741                         }
1742                         /* might be a policy change, too, so fall through */
1743                 }
1744                 dprintk("governor: change or update limits\n");
1745                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1746         }
1747
1748 error_out:
1749         cpufreq_debug_enable_ratelimit();
1750         return ret;
1751 }
1752
1753 /**
1754  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1755  *      @cpu: CPU which shall be re-evaluated
1756  *
1757  *      Usefull for policy notifiers which have different necessities
1758  *      at different times.
1759  */
1760 int cpufreq_update_policy(unsigned int cpu)
1761 {
1762         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1763         struct cpufreq_policy policy;
1764         int ret;
1765
1766         if (!data) {
1767                 ret = -ENODEV;
1768                 goto no_policy;
1769         }
1770
1771         if (unlikely(lock_policy_rwsem_write(cpu))) {
1772                 ret = -EINVAL;
1773                 goto fail;
1774         }
1775
1776         dprintk("updating policy for CPU %u\n", cpu);
1777         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1778         policy.min = data->user_policy.min;
1779         policy.max = data->user_policy.max;
1780         policy.policy = data->user_policy.policy;
1781         policy.governor = data->user_policy.governor;
1782
1783         /* BIOS might change freq behind our back
1784           -> ask driver for current freq and notify governors about a change */
1785         if (cpufreq_driver->get) {
1786                 policy.cur = cpufreq_driver->get(cpu);
1787                 if (!data->cur) {
1788                         dprintk("Driver did not initialize current freq");
1789                         data->cur = policy.cur;
1790                 } else {
1791                         if (data->cur != policy.cur)
1792                                 cpufreq_out_of_sync(cpu, data->cur,
1793                                                                 policy.cur);
1794                 }
1795         }
1796
1797         ret = __cpufreq_set_policy(data, &policy);
1798
1799         unlock_policy_rwsem_write(cpu);
1800
1801 fail:
1802         cpufreq_cpu_put(data);
1803 no_policy:
1804         return ret;
1805 }
1806 EXPORT_SYMBOL(cpufreq_update_policy);
1807
1808 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1809                                         unsigned long action, void *hcpu)
1810 {
1811         unsigned int cpu = (unsigned long)hcpu;
1812         struct sys_device *sys_dev;
1813
1814         sys_dev = get_cpu_sysdev(cpu);
1815         if (sys_dev) {
1816                 switch (action) {
1817                 case CPU_ONLINE:
1818                 case CPU_ONLINE_FROZEN:
1819                         cpufreq_add_dev(sys_dev);
1820                         break;
1821                 case CPU_DOWN_PREPARE:
1822                 case CPU_DOWN_PREPARE_FROZEN:
1823                         if (unlikely(lock_policy_rwsem_write(cpu)))
1824                                 BUG();
1825
1826                         __cpufreq_remove_dev(sys_dev);
1827                         break;
1828                 case CPU_DOWN_FAILED:
1829                 case CPU_DOWN_FAILED_FROZEN:
1830                         cpufreq_add_dev(sys_dev);
1831                         break;
1832                 }
1833         }
1834         return NOTIFY_OK;
1835 }
1836
1837 static struct notifier_block __refdata cpufreq_cpu_notifier =
1838 {
1839     .notifier_call = cpufreq_cpu_callback,
1840 };
1841
1842 /*********************************************************************
1843  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1844  *********************************************************************/
1845
1846 /**
1847  * cpufreq_register_driver - register a CPU Frequency driver
1848  * @driver_data: A struct cpufreq_driver containing the values#
1849  * submitted by the CPU Frequency driver.
1850  *
1851  *   Registers a CPU Frequency driver to this core code. This code
1852  * returns zero on success, -EBUSY when another driver got here first
1853  * (and isn't unregistered in the meantime).
1854  *
1855  */
1856 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1857 {
1858         unsigned long flags;
1859         int ret;
1860
1861         if (!driver_data || !driver_data->verify || !driver_data->init ||
1862             ((!driver_data->setpolicy) && (!driver_data->target)))
1863                 return -EINVAL;
1864
1865         dprintk("trying to register driver %s\n", driver_data->name);
1866
1867         if (driver_data->setpolicy)
1868                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1869
1870         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1871         if (cpufreq_driver) {
1872                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1873                 return -EBUSY;
1874         }
1875         cpufreq_driver = driver_data;
1876         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1877
1878         ret = sysdev_driver_register(&cpu_sysdev_class,
1879                                         &cpufreq_sysdev_driver);
1880
1881         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1882                 int i;
1883                 ret = -ENODEV;
1884
1885                 /* check for at least one working CPU */
1886                 for (i = 0; i < nr_cpu_ids; i++)
1887                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1888                                 ret = 0;
1889                                 break;
1890                         }
1891
1892                 /* if all ->init() calls failed, unregister */
1893                 if (ret) {
1894                         dprintk("no CPU initialized for driver %s\n",
1895                                                         driver_data->name);
1896                         sysdev_driver_unregister(&cpu_sysdev_class,
1897                                                 &cpufreq_sysdev_driver);
1898
1899                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1900                         cpufreq_driver = NULL;
1901                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1902                 }
1903         }
1904
1905         if (!ret) {
1906                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1907                 dprintk("driver %s up and running\n", driver_data->name);
1908                 cpufreq_debug_enable_ratelimit();
1909         }
1910
1911         return ret;
1912 }
1913 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1914
1915
1916 /**
1917  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1918  *
1919  *    Unregister the current CPUFreq driver. Only call this if you have
1920  * the right to do so, i.e. if you have succeeded in initialising before!
1921  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1922  * currently not initialised.
1923  */
1924 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1925 {
1926         unsigned long flags;
1927
1928         cpufreq_debug_disable_ratelimit();
1929
1930         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1931                 cpufreq_debug_enable_ratelimit();
1932                 return -EINVAL;
1933         }
1934
1935         dprintk("unregistering driver %s\n", driver->name);
1936
1937         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1938         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1939
1940         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1941         cpufreq_driver = NULL;
1942         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1943
1944         return 0;
1945 }
1946 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1947
1948 static int __init cpufreq_core_init(void)
1949 {
1950         int cpu;
1951
1952         for_each_possible_cpu(cpu) {
1953                 per_cpu(policy_cpu, cpu) = -1;
1954                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1955         }
1956
1957         cpufreq_global_kobject = kobject_create_and_add("cpufreq",
1958                                                 &cpu_sysdev_class.kset.kobj);
1959         BUG_ON(!cpufreq_global_kobject);
1960
1961         return 0;
1962 }
1963 core_initcall(cpufreq_core_init);