drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static bool init_cpufreq_transition_notifier_list_called;
 122 static int __init init_cpufreq_transition_notifier_list(void)
 123 {
 124         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 125         init_cpufreq_transition_notifier_list_called = true;
 126         return 0;
 127 }
 128 pure_initcall(init_cpufreq_transition_notifier_list);
 129
 130 static LIST_HEAD(cpufreq_governor_list);
 131 static DEFINE_MUTEX (cpufreq_governor_mutex);
 132
 133 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 134 {
 135         struct cpufreq_policy *data;
 136         unsigned long flags;
 137
 138         if (cpu >= nr_cpu_ids)
 139                 goto err_out;
 140
 141         /* get the cpufreq driver */
 142         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 143
 144         if (!cpufreq_driver)
 145                 goto err_out_unlock;
 146
 147         if (!try_module_get(cpufreq_driver->owner))
 148                 goto err_out_unlock;
 149
 150
 151         /* get the CPU */
 152         data = per_cpu(cpufreq_cpu_data, cpu);
 153
 154         if (!data)
 155                 goto err_out_put_module;
 156
 157         if (!kobject_get(&data->kobj))
 158                 goto err_out_put_module;
 159
 160         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 161         return data;
 162
 163 err_out_put_module:
 164         module_put(cpufreq_driver->owner);
 165 err_out_unlock:
 166         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 167 err_out:
 168         return NULL;
 169 }
 170 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 171
 172
 173 void cpufreq_cpu_put(struct cpufreq_policy *data)
 174 {
 175         kobject_put(&data->kobj);
 176         module_put(cpufreq_driver->owner);
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 179
 180
 181 /*********************************************************************
 182  *                     UNIFIED DEBUG HELPERS                         *
 183  *********************************************************************/
 184 #ifdef CONFIG_CPU_FREQ_DEBUG
 185
 186 /* what part(s) of the CPUfreq subsystem are debugged? */
 187 static unsigned int debug;
 188
 189 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 190  * set or modify this value.
 191  */
 192 static unsigned int debug_ratelimit = 1;
 193
 194 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 195  * loading of a cpufreq driver, temporarily disabled when a new policy
 196  * is set, and disabled upon cpufreq driver removal
 197  */
 198 static unsigned int disable_ratelimit = 1;
 199 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 200
 201 static void cpufreq_debug_enable_ratelimit(void)
 202 {
 203         unsigned long flags;
 204
 205         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 206         if (disable_ratelimit)
 207                 disable_ratelimit--;
 208         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 209 }
 210
 211 static void cpufreq_debug_disable_ratelimit(void)
 212 {
 213         unsigned long flags;
 214
 215         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 216         disable_ratelimit++;
 217         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 218 }
 219
 220 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 221                         const char *fmt, ...)
 222 {
 223         char s[256];
 224         va_list args;
 225         unsigned int len;
 226         unsigned long flags;
 227
 228         WARN_ON(!prefix);
 229         if (type & debug) {
 230                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 231                 if (!disable_ratelimit && debug_ratelimit
 232                                         && !printk_ratelimit()) {
 233                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 234                         return;
 235                 }
 236                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 237
 238                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 239
 240                 va_start(args, fmt);
 241                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 242                 va_end(args);
 243
 244                 printk(s);
 245
 246                 WARN_ON(len < 5);
 247         }
 248 }
 249 EXPORT_SYMBOL(cpufreq_debug_printk);
 250
 251
 252 module_param(debug, uint, 0644);
 253 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 254                         " 2 to debug drivers, and 4 to debug governors.");
 255
 256 module_param(debug_ratelimit, uint, 0644);
 257 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 258                                         " set to 0 to disable ratelimiting.");
 259
 260 #else /* !CONFIG_CPU_FREQ_DEBUG */
 261
 262 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 263 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 264
 265 #endif /* CONFIG_CPU_FREQ_DEBUG */
 266
 267
 268 /*********************************************************************
 269  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 270  *********************************************************************/
 271
 272 /**
 273  * adjust_jiffies - adjust the system "loops_per_jiffy"
 274  *
 275  * This function alters the system "loops_per_jiffy" for the clock
 276  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 277  * systems as each CPU might be scaled differently. So, use the arch
 278  * per-CPU loops_per_jiffy value wherever possible.
 279  */
 280 #ifndef CONFIG_SMP
 281 static unsigned long l_p_j_ref;
 282 static unsigned int  l_p_j_ref_freq;
 283
 284 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 285 {
 286         if (ci->flags & CPUFREQ_CONST_LOOPS)
 287                 return;
 288
 289         if (!l_p_j_ref_freq) {
 290                 l_p_j_ref = loops_per_jiffy;
 291                 l_p_j_ref_freq = ci->old;
 292                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 293                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 294         }
 295         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 296             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 297             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 298                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 299                                                                 ci->new);
 300                 dprintk("scaling loops_per_jiffy to %lu "
 301                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 302         }
 303 }
 304 #else
 305 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 306 {
 307         return;
 308 }
 309 #endif
 310
 311
 312 /**
 313  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 314  * on frequency transition.
 315  *
 316  * This function calls the transition notifiers and the "adjust_jiffies"
 317  * function. It is called twice on all CPU frequency changes that have
 318  * external effects.
 319  */
 320 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 321 {
 322         struct cpufreq_policy *policy;
 323
 324         BUG_ON(irqs_disabled());
 325
 326         freqs->flags = cpufreq_driver->flags;
 327         dprintk("notification %u of frequency transition to %u kHz\n",
 328                 state, freqs->new);
 329
 330         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 331         switch (state) {
 332
 333         case CPUFREQ_PRECHANGE:
 334                 /* detect if the driver reported a value as "old frequency"
 335                  * which is not equal to what the cpufreq core thinks is
 336                  * "old frequency".
 337                  */
 338                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 339                         if ((policy) && (policy->cpu == freqs->cpu) &&
 340                             (policy->cur) && (policy->cur != freqs->old)) {
 341                                 dprintk("Warning: CPU frequency is"
 342                                         " %u, cpufreq assumed %u kHz.\n",
 343                                         freqs->old, policy->cur);
 344                                 freqs->old = policy->cur;
 345                         }
 346                 }
 347                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 348                                 CPUFREQ_PRECHANGE, freqs);
 349                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 350                 break;
 351
 352         case CPUFREQ_POSTCHANGE:
 353                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 354                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 355                                 CPUFREQ_POSTCHANGE, freqs);
 356                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 357                         policy->cur = freqs->new;
 358                 break;
 359         }
 360 }
 361 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 362
 363
 364
 365 /*********************************************************************
 366  *                          SYSFS INTERFACE                          *
 367  *********************************************************************/
 368
 369 static struct cpufreq_governor *__find_governor(const char *str_governor)
 370 {
 371         struct cpufreq_governor *t;
 372
 373         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 374                 if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 375                         return t;
 376
 377         return NULL;
 378 }
 379
 380 /**
 381  * cpufreq_parse_governor - parse a governor string
 382  */
 383 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 384                                 struct cpufreq_governor **governor)
 385 {
 386         int err = -EINVAL;
 387
 388         if (!cpufreq_driver)
 389                 goto out;
 390
 391         if (cpufreq_driver->setpolicy) {
 392                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 393                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 394                         err = 0;
 395                 } else if (!strnicmp(str_governor, "powersave",
 396                                                 CPUFREQ_NAME_LEN)) {
 397                         *policy = CPUFREQ_POLICY_POWERSAVE;
 398                         err = 0;
 399                 }
 400         } else if (cpufreq_driver->target) {
 401                 struct cpufreq_governor *t;
 402
 403                 mutex_lock(&cpufreq_governor_mutex);
 404
 405                 t = __find_governor(str_governor);
 406
 407                 if (t == NULL) {
 408                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 409                                                                 str_governor);
 410
 411                         if (name) {
 412                                 int ret;
 413
 414                                 mutex_unlock(&cpufreq_governor_mutex);
 415                                 ret = request_module("%s", name);
 416                                 mutex_lock(&cpufreq_governor_mutex);
 417
 418                                 if (ret == 0)
 419                                         t = __find_governor(str_governor);
 420                         }
 421
 422                         kfree(name);
 423                 }
 424
 425                 if (t != NULL) {
 426                         *governor = t;
 427                         err = 0;
 428                 }
 429
 430                 mutex_unlock(&cpufreq_governor_mutex);
 431         }
 432   out:
 433         return err;
 434 }
 435
 436
 437 /* drivers/base/cpu.c */
 438 extern struct sysdev_class cpu_sysdev_class;
 439
 440
 441 /**
 442  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 443  * print out cpufreq information
 444  *
 445  * Write out information from cpufreq_driver->policy[cpu]; object must be
 446  * "unsigned int".
 447  */
 448
 449 #define show_one(file_name, object)                     \
 450 static ssize_t show_##file_name                         \
 451 (struct cpufreq_policy *policy, char *buf)              \
 452 {                                                       \
 453         return sprintf (buf, "%u\n", policy->object);   \
 454 }
 455
 456 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 457 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret = -EINVAL;                                     \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf (buf, "%u", &new_policy.object);                   \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq,min);
 490 store_one(scaling_max_freq,max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504
 505 /**
 506  * show_scaling_governor - show the current policy for the specified CPU
 507  */
 508 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 509 {
 510         if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 511                 return sprintf(buf, "powersave\n");
 512         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 513                 return sprintf(buf, "performance\n");
 514         else if (policy->governor)
 515                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 516         return -EINVAL;
 517 }
 518
 519
 520 /**
 521  * store_scaling_governor - store policy for the specified CPU
 522  */
 523 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 524                                         const char *buf, size_t count)
 525 {
 526         unsigned int ret = -EINVAL;
 527         char    str_governor[16];
 528         struct cpufreq_policy new_policy;
 529
 530         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 531         if (ret)
 532                 return ret;
 533
 534         ret = sscanf (buf, "%15s", str_governor);
 535         if (ret != 1)
 536                 return -EINVAL;
 537
 538         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 539                                                 &new_policy.governor))
 540                 return -EINVAL;
 541
 542         /* Do not use cpufreq_set_policy here or the user_policy.max
 543            will be wrongly overridden */
 544         ret = __cpufreq_set_policy(policy, &new_policy);
 545
 546         policy->user_policy.policy = policy->policy;
 547         policy->user_policy.governor = policy->governor;
 548
 549         if (ret)
 550                 return ret;
 551         else
 552                 return count;
 553 }
 554
 555 /**
 556  * show_scaling_driver - show the cpufreq driver currently loaded
 557  */
 558 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 559 {
 560         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 561 }
 562
 563 /**
 564  * show_scaling_available_governors - show the available CPUfreq governors
 565  */
 566 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 567                                                 char *buf)
 568 {
 569         ssize_t i = 0;
 570         struct cpufreq_governor *t;
 571
 572         if (!cpufreq_driver->target) {
 573                 i += sprintf(buf, "performance powersave");
 574                 goto out;
 575         }
 576
 577         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 578                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 579                         goto out;
 580                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581         }
 582 out:
 583         i += sprintf(&buf[i], "\n");
 584         return i;
 585 }
 586
 587 static ssize_t show_cpus(cpumask_t mask, char *buf)
 588 {
 589         ssize_t i = 0;
 590         unsigned int cpu;
 591
 592         for_each_cpu_mask_nr(cpu, mask) {
 593                 if (i)
 594                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 595                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 596                 if (i >= (PAGE_SIZE - 5))
 597                     break;
 598         }
 599         i += sprintf(&buf[i], "\n");
 600         return i;
 601 }
 602
 603 /**
 604  * show_related_cpus - show the CPUs affected by each transition even if
 605  * hw coordination is in use
 606  */
 607 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 608 {
 609         if (cpus_empty(policy->related_cpus))
 610                 return show_cpus(policy->cpus, buf);
 611         return show_cpus(policy->related_cpus, buf);
 612 }
 613
 614 /**
 615  * show_affected_cpus - show the CPUs affected by each transition
 616  */
 617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618 {
 619         return show_cpus(policy->cpus, buf);
 620 }
 621
 622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                         const char *buf, size_t count)
 624 {
 625         unsigned int freq = 0;
 626         unsigned int ret;
 627
 628         if (!policy->governor || !policy->governor->store_setspeed)
 629                 return -EINVAL;
 630
 631         ret = sscanf(buf, "%u", &freq);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         policy->governor->store_setspeed(policy, freq);
 636
 637         return count;
 638 }
 639
 640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641 {
 642         if (!policy->governor || !policy->governor->show_setspeed)
 643                 return sprintf(buf, "<unsupported>\n");
 644
 645         return policy->governor->show_setspeed(policy, buf);
 646 }
 647
 648 #define define_one_ro(_name) \
 649 static struct freq_attr _name = \
 650 __ATTR(_name, 0444, show_##_name, NULL)
 651
 652 #define define_one_ro0400(_name) \
 653 static struct freq_attr _name = \
 654 __ATTR(_name, 0400, show_##_name, NULL)
 655
 656 #define define_one_rw(_name) \
 657 static struct freq_attr _name = \
 658 __ATTR(_name, 0644, show_##_name, store_##_name)
 659
 660 define_one_ro0400(cpuinfo_cur_freq);
 661 define_one_ro(cpuinfo_min_freq);
 662 define_one_ro(cpuinfo_max_freq);
 663 define_one_ro(scaling_available_governors);
 664 define_one_ro(scaling_driver);
 665 define_one_ro(scaling_cur_freq);
 666 define_one_ro(related_cpus);
 667 define_one_ro(affected_cpus);
 668 define_one_rw(scaling_min_freq);
 669 define_one_rw(scaling_max_freq);
 670 define_one_rw(scaling_governor);
 671 define_one_rw(scaling_setspeed);
 672
 673 static struct attribute *default_attrs[] = {
 674         &cpuinfo_min_freq.attr,
 675         &cpuinfo_max_freq.attr,
 676         &scaling_min_freq.attr,
 677         &scaling_max_freq.attr,
 678         &affected_cpus.attr,
 679         &related_cpus.attr,
 680         &scaling_governor.attr,
 681         &scaling_driver.attr,
 682         &scaling_available_governors.attr,
 683         &scaling_setspeed.attr,
 684         NULL
 685 };
 686
 687 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 688 #define to_attr(a) container_of(a,struct freq_attr,attr)
 689
 690 static ssize_t show(struct kobject *kobj, struct attribute *attr ,char *buf)
 691 {
 692         struct cpufreq_policy *policy = to_policy(kobj);
 693         struct freq_attr *fattr = to_attr(attr);
 694         ssize_t ret = -EINVAL;
 695         policy = cpufreq_cpu_get(policy->cpu);
 696         if (!policy)
 697                 goto no_policy;
 698
 699         if (lock_policy_rwsem_read(policy->cpu) < 0)
 700                 goto fail;
 701
 702         if (fattr->show)
 703                 ret = fattr->show(policy, buf);
 704         else
 705                 ret = -EIO;
 706
 707         unlock_policy_rwsem_read(policy->cpu);
 708 fail:
 709         cpufreq_cpu_put(policy);
 710 no_policy:
 711         return ret;
 712 }
 713
 714 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 715                      const char *buf, size_t count)
 716 {
 717         struct cpufreq_policy *policy = to_policy(kobj);
 718         struct freq_attr *fattr = to_attr(attr);
 719         ssize_t ret = -EINVAL;
 720         policy = cpufreq_cpu_get(policy->cpu);
 721         if (!policy)
 722                 goto no_policy;
 723
 724         if (lock_policy_rwsem_write(policy->cpu) < 0)
 725                 goto fail;
 726
 727         if (fattr->store)
 728                 ret = fattr->store(policy, buf, count);
 729         else
 730                 ret = -EIO;
 731
 732         unlock_policy_rwsem_write(policy->cpu);
 733 fail:
 734         cpufreq_cpu_put(policy);
 735 no_policy:
 736         return ret;
 737 }
 738
 739 static void cpufreq_sysfs_release(struct kobject *kobj)
 740 {
 741         struct cpufreq_policy *policy = to_policy(kobj);
 742         dprintk("last reference is dropped\n");
 743         complete(&policy->kobj_unregister);
 744 }
 745
 746 static struct sysfs_ops sysfs_ops = {
 747         .show   = show,
 748         .store  = store,
 749 };
 750
 751 static struct kobj_type ktype_cpufreq = {
 752         .sysfs_ops      = &sysfs_ops,
 753         .default_attrs  = default_attrs,
 754         .release        = cpufreq_sysfs_release,
 755 };
 756
 757 static struct kobj_type ktype_empty_cpufreq = {
 758         .sysfs_ops      = &sysfs_ops,
 759         .release        = cpufreq_sysfs_release,
 760 };
 761
 762
 763 /**
 764  * cpufreq_add_dev - add a CPU device
 765  *
 766  * Adds the cpufreq interface for a CPU device.
 767  */
 768 static int cpufreq_add_dev(struct sys_device *sys_dev)
 769 {
 770         unsigned int cpu = sys_dev->id;
 771         int ret = 0;
 772         struct cpufreq_policy new_policy;
 773         struct cpufreq_policy *policy;
 774         struct freq_attr **drv_attr;
 775         struct sys_device *cpu_sys_dev;
 776         unsigned long flags;
 777         unsigned int j;
 778 #ifdef CONFIG_SMP
 779         struct cpufreq_policy *managed_policy;
 780 #endif
 781
 782         if (cpu_is_offline(cpu))
 783                 return 0;
 784
 785         cpufreq_debug_disable_ratelimit();
 786         dprintk("adding CPU %u\n", cpu);
 787
 788 #ifdef CONFIG_SMP
 789         /* check whether a different CPU already registered this
 790          * CPU because it is in the same boat. */
 791         policy = cpufreq_cpu_get(cpu);
 792         if (unlikely(policy)) {
 793                 cpufreq_cpu_put(policy);
 794                 cpufreq_debug_enable_ratelimit();
 795                 return 0;
 796         }
 797 #endif
 798
 799         if (!try_module_get(cpufreq_driver->owner)) {
 800                 ret = -EINVAL;
 801                 goto module_out;
 802         }
 803
 804         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 805         if (!policy) {
 806                 ret = -ENOMEM;
 807                 goto nomem_out;
 808         }
 809
 810         policy->cpu = cpu;
 811         policy->cpus = cpumask_of_cpu(cpu);
 812
 813         /* Initially set CPU itself as the policy_cpu */
 814         per_cpu(policy_cpu, cpu) = cpu;
 815         lock_policy_rwsem_write(cpu);
 816
 817         init_completion(&policy->kobj_unregister);
 818         INIT_WORK(&policy->update, handle_update);
 819
 820         /* Set governor before ->init, so that driver could check it */
 821         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 822         /* call driver. From then on the cpufreq must be able
 823          * to accept all calls to ->verify and ->setpolicy for this CPU
 824          */
 825         ret = cpufreq_driver->init(policy);
 826         if (ret) {
 827                 dprintk("initialization failed\n");
 828                 goto err_out;
 829         }
 830         policy->user_policy.min = policy->min;
 831         policy->user_policy.max = policy->max;
 832
 833         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 834                                      CPUFREQ_START, policy);
 835
 836 #ifdef CONFIG_SMP
 837
 838 #ifdef CONFIG_HOTPLUG_CPU
 839         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 840                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 841                 dprintk("Restoring governor %s for cpu %d\n",
 842                        policy->governor->name, cpu);
 843         }
 844 #endif
 845
 846         for_each_cpu_mask_nr(j, policy->cpus) {
 847                 if (cpu == j)
 848                         continue;
 849
 850                 /* check for existing affected CPUs.  They may not be aware
 851                  * of it due to CPU Hotplug.
 852                  */
 853                 managed_policy = cpufreq_cpu_get(j);            // FIXME: Where is this released?  What about error paths?
 854                 if (unlikely(managed_policy)) {
 855
 856                         /* Set proper policy_cpu */
 857                         unlock_policy_rwsem_write(cpu);
 858                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 859
 860                         if (lock_policy_rwsem_write(cpu) < 0)
 861                                 goto err_out_driver_exit;
 862
 863                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 864                         managed_policy->cpus = policy->cpus;
 865                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 866                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 867
 868                         dprintk("CPU already managed, adding link\n");
 869                         ret = sysfs_create_link(&sys_dev->kobj,
 870                                                 &managed_policy->kobj,
 871                                                 "cpufreq");
 872                         if (ret)
 873                                 goto err_out_driver_exit;
 874
 875                         cpufreq_debug_enable_ratelimit();
 876                         ret = 0;
 877                         goto err_out_driver_exit; /* call driver->exit() */
 878                 }
 879         }
 880 #endif
 881         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 882
 883         /* prepare interface data */
 884         if (!cpufreq_driver->hide_interface) {
 885                 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 886                                            &sys_dev->kobj, "cpufreq");
 887                 if (ret)
 888                         goto err_out_driver_exit;
 889
 890                 /* set up files for this cpu device */
 891                 drv_attr = cpufreq_driver->attr;
 892                 while ((drv_attr) && (*drv_attr)) {
 893                         ret = sysfs_create_file(&policy->kobj,
 894                                                 &((*drv_attr)->attr));
 895                         if (ret)
 896                                 goto err_out_driver_exit;
 897                         drv_attr++;
 898                 }
 899                 if (cpufreq_driver->get) {
 900                         ret = sysfs_create_file(&policy->kobj,
 901                                                 &cpuinfo_cur_freq.attr);
 902                         if (ret)
 903                                 goto err_out_driver_exit;
 904                 }
 905                 if (cpufreq_driver->target) {
 906                         ret = sysfs_create_file(&policy->kobj,
 907                                                 &scaling_cur_freq.attr);
 908                         if (ret)
 909                                 goto err_out_driver_exit;
 910                 }
 911         } else {
 912                 ret = kobject_init_and_add(&policy->kobj, &ktype_empty_cpufreq,
 913                                            &sys_dev->kobj, "cpufreq");
 914                 if (ret)
 915                         goto err_out_driver_exit;
 916         }
 917
 918         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 919         for_each_cpu_mask_nr(j, policy->cpus) {
 920                 per_cpu(cpufreq_cpu_data, j) = policy;
 921                 per_cpu(policy_cpu, j) = policy->cpu;
 922         }
 923         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 924
 925         /* symlink affected CPUs */
 926         for_each_cpu_mask_nr(j, policy->cpus) {
 927                 if (j == cpu)
 928                         continue;
 929                 if (!cpu_online(j))
 930                         continue;
 931
 932                 dprintk("CPU %u already managed, adding link\n", j);
 933                 cpufreq_cpu_get(cpu);
 934                 cpu_sys_dev = get_cpu_sysdev(j);
 935                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 936                                         "cpufreq");
 937                 if (ret)
 938                         goto err_out_unregister;
 939         }
 940
 941         policy->governor = NULL; /* to assure that the starting sequence is
 942                                   * run in cpufreq_set_policy */
 943
 944         /* set default policy */
 945         ret = __cpufreq_set_policy(policy, &new_policy);
 946         policy->user_policy.policy = policy->policy;
 947         policy->user_policy.governor = policy->governor;
 948
 949         if (ret) {
 950                 dprintk("setting policy failed\n");
 951                 goto err_out_unregister;
 952         }
 953
 954         unlock_policy_rwsem_write(cpu);
 955
 956         kobject_uevent(&policy->kobj, KOBJ_ADD);
 957         module_put(cpufreq_driver->owner);
 958         dprintk("initialization complete\n");
 959         cpufreq_debug_enable_ratelimit();
 960
 961         return 0;
 962
 963
 964 err_out_unregister:
 965         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 966         for_each_cpu_mask_nr(j, policy->cpus)
 967                 per_cpu(cpufreq_cpu_data, j) = NULL;
 968         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 969
 970         kobject_put(&policy->kobj);
 971         wait_for_completion(&policy->kobj_unregister);
 972
 973 err_out_driver_exit:
 974         if (cpufreq_driver->exit)
 975                 cpufreq_driver->exit(policy);
 976
 977 err_out:
 978         unlock_policy_rwsem_write(cpu);
 979         kfree(policy);
 980
 981 nomem_out:
 982         module_put(cpufreq_driver->owner);
 983 module_out:
 984         cpufreq_debug_enable_ratelimit();
 985         return ret;
 986 }
 987
 988
 989 /**
 990  * __cpufreq_remove_dev - remove a CPU device
 991  *
 992  * Removes the cpufreq interface for a CPU device.
 993  * Caller should already have policy_rwsem in write mode for this CPU.
 994  * This routine frees the rwsem before returning.
 995  */
 996 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
 997 {
 998         unsigned int cpu = sys_dev->id;
 999         unsigned long flags;
1000         struct cpufreq_policy *data;
1001 #ifdef CONFIG_SMP
1002         struct sys_device *cpu_sys_dev;
1003         unsigned int j;
1004 #endif
1005
1006         cpufreq_debug_disable_ratelimit();
1007         dprintk("unregistering CPU %u\n", cpu);
1008
1009         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1010         data = per_cpu(cpufreq_cpu_data, cpu);
1011
1012         if (!data) {
1013                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1014                 cpufreq_debug_enable_ratelimit();
1015                 unlock_policy_rwsem_write(cpu);
1016                 return -EINVAL;
1017         }
1018         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1019
1020
1021 #ifdef CONFIG_SMP
1022         /* if this isn't the CPU which is the parent of the kobj, we
1023          * only need to unlink, put and exit
1024          */
1025         if (unlikely(cpu != data->cpu)) {
1026                 dprintk("removing link\n");
1027                 cpu_clear(cpu, data->cpus);
1028                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1029                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1030                 cpufreq_cpu_put(data);
1031                 cpufreq_debug_enable_ratelimit();
1032                 unlock_policy_rwsem_write(cpu);
1033                 return 0;
1034         }
1035 #endif
1036
1037 #ifdef CONFIG_SMP
1038
1039 #ifdef CONFIG_HOTPLUG_CPU
1040         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1041 #endif
1042
1043         /* if we have other CPUs still registered, we need to unlink them,
1044          * or else wait_for_completion below will lock up. Clean the
1045          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1046          * the sysfs links afterwards.
1047          */
1048         if (unlikely(cpus_weight(data->cpus) > 1)) {
1049                 for_each_cpu_mask_nr(j, data->cpus) {
1050                         if (j == cpu)
1051                                 continue;
1052                         per_cpu(cpufreq_cpu_data, j) = NULL;
1053                 }
1054         }
1055
1056         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1057
1058         if (unlikely(cpus_weight(data->cpus) > 1)) {
1059                 for_each_cpu_mask_nr(j, data->cpus) {
1060                         if (j == cpu)
1061                                 continue;
1062                         dprintk("removing link for cpu %u\n", j);
1063 #ifdef CONFIG_HOTPLUG_CPU
1064                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1065 #endif
1066                         cpu_sys_dev = get_cpu_sysdev(j);
1067                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1068                         cpufreq_cpu_put(data);
1069                 }
1070         }
1071 #else
1072         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1073 #endif
1074
1075         if (cpufreq_driver->target)
1076                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1077
1078         unlock_policy_rwsem_write(cpu);
1079
1080         kobject_put(&data->kobj);
1081
1082         /* we need to make sure that the underlying kobj is actually
1083          * not referenced anymore by anybody before we proceed with
1084          * unloading.
1085          */
1086         dprintk("waiting for dropping of refcount\n");
1087         wait_for_completion(&data->kobj_unregister);
1088         dprintk("wait complete\n");
1089
1090         if (cpufreq_driver->exit)
1091                 cpufreq_driver->exit(data);
1092
1093         kfree(data);
1094
1095         cpufreq_debug_enable_ratelimit();
1096         return 0;
1097 }
1098
1099
1100 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1101 {
1102         unsigned int cpu = sys_dev->id;
1103         int retval;
1104
1105         if (cpu_is_offline(cpu))
1106                 return 0;
1107
1108         if (unlikely(lock_policy_rwsem_write(cpu)))
1109                 BUG();
1110
1111         retval = __cpufreq_remove_dev(sys_dev);
1112         return retval;
1113 }
1114
1115
1116 static void handle_update(struct work_struct *work)
1117 {
1118         struct cpufreq_policy *policy =
1119                 container_of(work, struct cpufreq_policy, update);
1120         unsigned int cpu = policy->cpu;
1121         dprintk("handle_update for cpu %u called\n", cpu);
1122         cpufreq_update_policy(cpu);
1123 }
1124
1125 /**
1126  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1127  *      @cpu: cpu number
1128  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1129  *      @new_freq: CPU frequency the CPU actually runs at
1130  *
1131  *      We adjust to current frequency first, and need to clean up later. So either call
1132  *      to cpufreq_update_policy() or schedule handle_update()).
1133  */
1134 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1135                                 unsigned int new_freq)
1136 {
1137         struct cpufreq_freqs freqs;
1138
1139         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1140                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1141
1142         freqs.cpu = cpu;
1143         freqs.old = old_freq;
1144         freqs.new = new_freq;
1145         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1146         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1147 }
1148
1149
1150 /**
1151  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1152  * @cpu: CPU number
1153  *
1154  * This is the last known freq, without actually getting it from the driver.
1155  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1156  */
1157 unsigned int cpufreq_quick_get(unsigned int cpu)
1158 {
1159         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1160         unsigned int ret_freq = 0;
1161
1162         if (policy) {
1163                 ret_freq = policy->cur;
1164                 cpufreq_cpu_put(policy);
1165         }
1166
1167         return ret_freq;
1168 }
1169 EXPORT_SYMBOL(cpufreq_quick_get);
1170
1171
1172 static unsigned int __cpufreq_get(unsigned int cpu)
1173 {
1174         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1175         unsigned int ret_freq = 0;
1176
1177         if (!cpufreq_driver->get)
1178                 return ret_freq;
1179
1180         ret_freq = cpufreq_driver->get(cpu);
1181
1182         if (ret_freq && policy->cur &&
1183                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1184                 /* verify no discrepancy between actual and
1185                                         saved value exists */
1186                 if (unlikely(ret_freq != policy->cur)) {
1187                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1188                         schedule_work(&policy->update);
1189                 }
1190         }
1191
1192         return ret_freq;
1193 }
1194
1195 /**
1196  * cpufreq_get - get the current CPU frequency (in kHz)
1197  * @cpu: CPU number
1198  *
1199  * Get the CPU current (static) CPU frequency
1200  */
1201 unsigned int cpufreq_get(unsigned int cpu)
1202 {
1203         unsigned int ret_freq = 0;
1204         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1205
1206         if (!policy)
1207                 goto out;
1208
1209         if (unlikely(lock_policy_rwsem_read(cpu)))
1210                 goto out_policy;
1211
1212         ret_freq = __cpufreq_get(cpu);
1213
1214         unlock_policy_rwsem_read(cpu);
1215
1216 out_policy:
1217         cpufreq_cpu_put(policy);
1218 out:
1219         return ret_freq;
1220 }
1221 EXPORT_SYMBOL(cpufreq_get);
1222
1223
1224 /**
1225  *      cpufreq_suspend - let the low level driver prepare for suspend
1226  */
1227
1228 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1229 {
1230         int cpu = sysdev->id;
1231         int ret = 0;
1232         unsigned int cur_freq = 0;
1233         struct cpufreq_policy *cpu_policy;
1234
1235         dprintk("suspending cpu %u\n", cpu);
1236
1237         if (!cpu_online(cpu))
1238                 return 0;
1239
1240         /* we may be lax here as interrupts are off. Nonetheless
1241          * we need to grab the correct cpu policy, as to check
1242          * whether we really run on this CPU.
1243          */
1244
1245         cpu_policy = cpufreq_cpu_get(cpu);
1246         if (!cpu_policy)
1247                 return -EINVAL;
1248
1249         /* only handle each CPU group once */
1250         if (unlikely(cpu_policy->cpu != cpu))
1251                 goto out;
1252
1253         if (cpufreq_driver->suspend) {
1254                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1255                 if (ret) {
1256                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1257                                         "step on CPU %u\n", cpu_policy->cpu);
1258                         goto out;
1259                 }
1260         }
1261
1262         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1263                 goto out;
1264
1265         if (cpufreq_driver->get)
1266                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1267
1268         if (!cur_freq || !cpu_policy->cur) {
1269                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1270                        "frequency is what timing core thinks it is.\n");
1271                 goto out;
1272         }
1273
1274         if (unlikely(cur_freq != cpu_policy->cur)) {
1275                 struct cpufreq_freqs freqs;
1276
1277                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1278                         dprintk("Warning: CPU frequency is %u, "
1279                                "cpufreq assumed %u kHz.\n",
1280                                cur_freq, cpu_policy->cur);
1281
1282                 freqs.cpu = cpu;
1283                 freqs.old = cpu_policy->cur;
1284                 freqs.new = cur_freq;
1285
1286                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1287                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1288                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1289
1290                 cpu_policy->cur = cur_freq;
1291         }
1292
1293 out:
1294         cpufreq_cpu_put(cpu_policy);
1295         return ret;
1296 }
1297
1298 /**
1299  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1300  *
1301  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1302  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1303  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1304  *          restored.
1305  */
1306 static int cpufreq_resume(struct sys_device *sysdev)
1307 {
1308         int cpu = sysdev->id;
1309         int ret = 0;
1310         struct cpufreq_policy *cpu_policy;
1311
1312         dprintk("resuming cpu %u\n", cpu);
1313
1314         if (!cpu_online(cpu))
1315                 return 0;
1316
1317         /* we may be lax here as interrupts are off. Nonetheless
1318          * we need to grab the correct cpu policy, as to check
1319          * whether we really run on this CPU.
1320          */
1321
1322         cpu_policy = cpufreq_cpu_get(cpu);
1323         if (!cpu_policy)
1324                 return -EINVAL;
1325
1326         /* only handle each CPU group once */
1327         if (unlikely(cpu_policy->cpu != cpu))
1328                 goto fail;
1329
1330         if (cpufreq_driver->resume) {
1331                 ret = cpufreq_driver->resume(cpu_policy);
1332                 if (ret) {
1333                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1334                                         "step on CPU %u\n", cpu_policy->cpu);
1335                         goto fail;
1336                 }
1337         }
1338
1339         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1340                 unsigned int cur_freq = 0;
1341
1342                 if (cpufreq_driver->get)
1343                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1344
1345                 if (!cur_freq || !cpu_policy->cur) {
1346                         printk(KERN_ERR "cpufreq: resume failed to assert "
1347                                         "current frequency is what timing core "
1348                                         "thinks it is.\n");
1349                         goto out;
1350                 }
1351
1352                 if (unlikely(cur_freq != cpu_policy->cur)) {
1353                         struct cpufreq_freqs freqs;
1354
1355                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1356                                 dprintk("Warning: CPU frequency "
1357                                        "is %u, cpufreq assumed %u kHz.\n",
1358                                        cur_freq, cpu_policy->cur);
1359
1360                         freqs.cpu = cpu;
1361                         freqs.old = cpu_policy->cur;
1362                         freqs.new = cur_freq;
1363
1364                         srcu_notifier_call_chain(
1365                                         &cpufreq_transition_notifier_list,
1366                                         CPUFREQ_RESUMECHANGE, &freqs);
1367                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1368
1369                         cpu_policy->cur = cur_freq;
1370                 }
1371         }
1372
1373 out:
1374         schedule_work(&cpu_policy->update);
1375 fail:
1376         cpufreq_cpu_put(cpu_policy);
1377         return ret;
1378 }
1379
1380 static struct sysdev_driver cpufreq_sysdev_driver = {
1381         .add            = cpufreq_add_dev,
1382         .remove         = cpufreq_remove_dev,
1383         .suspend        = cpufreq_suspend,
1384         .resume         = cpufreq_resume,
1385 };
1386
1387
1388 /*********************************************************************
1389  *                     NOTIFIER LISTS INTERFACE                      *
1390  *********************************************************************/
1391
1392 /**
1393  *      cpufreq_register_notifier - register a driver with cpufreq
1394  *      @nb: notifier function to register
1395  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1396  *
1397  *      Add a driver to one of two lists: either a list of drivers that
1398  *      are notified about clock rate changes (once before and once after
1399  *      the transition), or a list of drivers that are notified about
1400  *      changes in cpufreq policy.
1401  *
1402  *      This function may sleep, and has the same return conditions as
1403  *      blocking_notifier_chain_register.
1404  */
1405 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1406 {
1407         int ret;
1408
1409         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1410
1411         switch (list) {
1412         case CPUFREQ_TRANSITION_NOTIFIER:
1413                 ret = srcu_notifier_chain_register(
1414                                 &cpufreq_transition_notifier_list, nb);
1415                 break;
1416         case CPUFREQ_POLICY_NOTIFIER:
1417                 ret = blocking_notifier_chain_register(
1418                                 &cpufreq_policy_notifier_list, nb);
1419                 break;
1420         default:
1421                 ret = -EINVAL;
1422         }
1423
1424         return ret;
1425 }
1426 EXPORT_SYMBOL(cpufreq_register_notifier);
1427
1428
1429 /**
1430  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1431  *      @nb: notifier block to be unregistered
1432  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1433  *
1434  *      Remove a driver from the CPU frequency notifier list.
1435  *
1436  *      This function may sleep, and has the same return conditions as
1437  *      blocking_notifier_chain_unregister.
1438  */
1439 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1440 {
1441         int ret;
1442
1443         switch (list) {
1444         case CPUFREQ_TRANSITION_NOTIFIER:
1445                 ret = srcu_notifier_chain_unregister(
1446                                 &cpufreq_transition_notifier_list, nb);
1447                 break;
1448         case CPUFREQ_POLICY_NOTIFIER:
1449                 ret = blocking_notifier_chain_unregister(
1450                                 &cpufreq_policy_notifier_list, nb);
1451                 break;
1452         default:
1453                 ret = -EINVAL;
1454         }
1455
1456         return ret;
1457 }
1458 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1459
1460
1461 /*********************************************************************
1462  *                              GOVERNORS                            *
1463  *********************************************************************/
1464
1465
1466 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1467                             unsigned int target_freq,
1468                             unsigned int relation)
1469 {
1470         int retval = -EINVAL;
1471
1472         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1473                 target_freq, relation);
1474         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1475                 retval = cpufreq_driver->target(policy, target_freq, relation);
1476
1477         return retval;
1478 }
1479 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1480
1481 int cpufreq_driver_target(struct cpufreq_policy *policy,
1482                           unsigned int target_freq,
1483                           unsigned int relation)
1484 {
1485         int ret = -EINVAL;
1486
1487         policy = cpufreq_cpu_get(policy->cpu);
1488         if (!policy)
1489                 goto no_policy;
1490
1491         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1492                 goto fail;
1493
1494         ret = __cpufreq_driver_target(policy, target_freq, relation);
1495
1496         unlock_policy_rwsem_write(policy->cpu);
1497
1498 fail:
1499         cpufreq_cpu_put(policy);
1500 no_policy:
1501         return ret;
1502 }
1503 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1504
1505 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1506 {
1507         int ret = 0;
1508
1509         policy = cpufreq_cpu_get(policy->cpu);
1510         if (!policy)
1511                 return -EINVAL;
1512
1513         if (cpu_online(cpu) && cpufreq_driver->getavg)
1514                 ret = cpufreq_driver->getavg(policy, cpu);
1515
1516         cpufreq_cpu_put(policy);
1517         return ret;
1518 }
1519 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1520
1521 /*
1522  * when "event" is CPUFREQ_GOV_LIMITS
1523  */
1524
1525 static int __cpufreq_governor(struct cpufreq_policy *policy,
1526                                         unsigned int event)
1527 {
1528         int ret;
1529
1530         /* Only must be defined when default governor is known to have latency
1531            restrictions, like e.g. conservative or ondemand.
1532            That this is the case is already ensured in Kconfig
1533         */
1534 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1535         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1536 #else
1537         struct cpufreq_governor *gov = NULL;
1538 #endif
1539
1540         if (policy->governor->max_transition_latency &&
1541             policy->cpuinfo.transition_latency >
1542             policy->governor->max_transition_latency) {
1543                 if (!gov)
1544                         return -EINVAL;
1545                 else {
1546                         printk(KERN_WARNING "%s governor failed, too long"
1547                                " transition latency of HW, fallback"
1548                                " to %s governor\n",
1549                                policy->governor->name,
1550                                gov->name);
1551                         policy->governor = gov;
1552                 }
1553         }
1554
1555         if (!try_module_get(policy->governor->owner))
1556                 return -EINVAL;
1557
1558         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1559                                                 policy->cpu, event);
1560         ret = policy->governor->governor(policy, event);
1561
1562         /* we keep one module reference alive for
1563                         each CPU governed by this CPU */
1564         if ((event != CPUFREQ_GOV_START) || ret)
1565                 module_put(policy->governor->owner);
1566         if ((event == CPUFREQ_GOV_STOP) && !ret)
1567                 module_put(policy->governor->owner);
1568
1569         return ret;
1570 }
1571
1572
1573 int cpufreq_register_governor(struct cpufreq_governor *governor)
1574 {
1575         int err;
1576
1577         if (!governor)
1578                 return -EINVAL;
1579
1580         mutex_lock(&cpufreq_governor_mutex);
1581
1582         err = -EBUSY;
1583         if (__find_governor(governor->name) == NULL) {
1584                 err = 0;
1585                 list_add(&governor->governor_list, &cpufreq_governor_list);
1586         }
1587
1588         mutex_unlock(&cpufreq_governor_mutex);
1589         return err;
1590 }
1591 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1592
1593
1594 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1595 {
1596         if (!governor)
1597                 return;
1598
1599         mutex_lock(&cpufreq_governor_mutex);
1600         list_del(&governor->governor_list);
1601         mutex_unlock(&cpufreq_governor_mutex);
1602         return;
1603 }
1604 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1605
1606
1607
1608 /*********************************************************************
1609  *                          POLICY INTERFACE                         *
1610  *********************************************************************/
1611
1612 /**
1613  * cpufreq_get_policy - get the current cpufreq_policy
1614  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1615  *
1616  * Reads the current cpufreq policy.
1617  */
1618 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1619 {
1620         struct cpufreq_policy *cpu_policy;
1621         if (!policy)
1622                 return -EINVAL;
1623
1624         cpu_policy = cpufreq_cpu_get(cpu);
1625         if (!cpu_policy)
1626                 return -EINVAL;
1627
1628         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1629
1630         cpufreq_cpu_put(cpu_policy);
1631         return 0;
1632 }
1633 EXPORT_SYMBOL(cpufreq_get_policy);
1634
1635
1636 /*
1637  * data   : current policy.
1638  * policy : policy to be set.
1639  */
1640 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1641                                 struct cpufreq_policy *policy)
1642 {
1643         int ret = 0;
1644
1645         cpufreq_debug_disable_ratelimit();
1646         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1647                 policy->min, policy->max);
1648
1649         memcpy(&policy->cpuinfo, &data->cpuinfo,
1650                                 sizeof(struct cpufreq_cpuinfo));
1651
1652         if (policy->min > data->max || policy->max < data->min) {
1653                 ret = -EINVAL;
1654                 goto error_out;
1655         }
1656
1657         /* verify the cpu speed can be set within this limit */
1658         ret = cpufreq_driver->verify(policy);
1659         if (ret)
1660                 goto error_out;
1661
1662         /* adjust if necessary - all reasons */
1663         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1664                         CPUFREQ_ADJUST, policy);
1665
1666         /* adjust if necessary - hardware incompatibility*/
1667         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1668                         CPUFREQ_INCOMPATIBLE, policy);
1669
1670         /* verify the cpu speed can be set within this limit,
1671            which might be different to the first one */
1672         ret = cpufreq_driver->verify(policy);
1673         if (ret)
1674                 goto error_out;
1675
1676         /* notification of the new policy */
1677         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1678                         CPUFREQ_NOTIFY, policy);
1679
1680         data->min = policy->min;
1681         data->max = policy->max;
1682
1683         dprintk("new min and max freqs are %u - %u kHz\n",
1684                                         data->min, data->max);
1685
1686         if (cpufreq_driver->setpolicy) {
1687                 data->policy = policy->policy;
1688                 dprintk("setting range\n");
1689                 ret = cpufreq_driver->setpolicy(policy);
1690         } else {
1691                 if (policy->governor != data->governor) {
1692                         /* save old, working values */
1693                         struct cpufreq_governor *old_gov = data->governor;
1694
1695                         dprintk("governor switch\n");
1696
1697                         /* end old governor */
1698                         if (data->governor)
1699                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1700
1701                         /* start new governor */
1702                         data->governor = policy->governor;
1703                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1704                                 /* new governor failed, so re-start old one */
1705                                 dprintk("starting governor %s failed\n",
1706                                                         data->governor->name);
1707                                 if (old_gov) {
1708                                         data->governor = old_gov;
1709                                         __cpufreq_governor(data,
1710                                                            CPUFREQ_GOV_START);
1711                                 }
1712                                 ret = -EINVAL;
1713                                 goto error_out;
1714                         }
1715                         /* might be a policy change, too, so fall through */
1716                 }
1717                 dprintk("governor: change or update limits\n");
1718                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1719         }
1720
1721 error_out:
1722         cpufreq_debug_enable_ratelimit();
1723         return ret;
1724 }
1725
1726 /**
1727  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1728  *      @cpu: CPU which shall be re-evaluated
1729  *
1730  *      Usefull for policy notifiers which have different necessities
1731  *      at different times.
1732  */
1733 int cpufreq_update_policy(unsigned int cpu)
1734 {
1735         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1736         struct cpufreq_policy policy;
1737         int ret;
1738
1739         if (!data) {
1740                 ret = -ENODEV;
1741                 goto no_policy;
1742         }
1743
1744         if (unlikely(lock_policy_rwsem_write(cpu))) {
1745                 ret = -EINVAL;
1746                 goto fail;
1747         }
1748
1749         dprintk("updating policy for CPU %u\n", cpu);
1750         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1751         policy.min = data->user_policy.min;
1752         policy.max = data->user_policy.max;
1753         policy.policy = data->user_policy.policy;
1754         policy.governor = data->user_policy.governor;
1755
1756         /* BIOS might change freq behind our back
1757           -> ask driver for current freq and notify governors about a change */
1758         if (cpufreq_driver->get) {
1759                 policy.cur = cpufreq_driver->get(cpu);
1760                 if (!data->cur) {
1761                         dprintk("Driver did not initialize current freq");
1762                         data->cur = policy.cur;
1763                 } else {
1764                         if (data->cur != policy.cur)
1765                                 cpufreq_out_of_sync(cpu, data->cur,
1766                                                                 policy.cur);
1767                 }
1768         }
1769
1770         ret = __cpufreq_set_policy(data, &policy);
1771
1772         unlock_policy_rwsem_write(cpu);
1773
1774 fail:
1775         cpufreq_cpu_put(data);
1776 no_policy:
1777         return ret;
1778 }
1779 EXPORT_SYMBOL(cpufreq_update_policy);
1780
1781 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1782                                         unsigned long action, void *hcpu)
1783 {
1784         unsigned int cpu = (unsigned long)hcpu;
1785         struct sys_device *sys_dev;
1786
1787         sys_dev = get_cpu_sysdev(cpu);
1788         if (sys_dev) {
1789                 switch (action) {
1790                 case CPU_ONLINE:
1791                 case CPU_ONLINE_FROZEN:
1792                         cpufreq_add_dev(sys_dev);
1793                         break;
1794                 case CPU_DOWN_PREPARE:
1795                 case CPU_DOWN_PREPARE_FROZEN:
1796                         if (unlikely(lock_policy_rwsem_write(cpu)))
1797                                 BUG();
1798
1799                         __cpufreq_remove_dev(sys_dev);
1800                         break;
1801                 case CPU_DOWN_FAILED:
1802                 case CPU_DOWN_FAILED_FROZEN:
1803                         cpufreq_add_dev(sys_dev);
1804                         break;
1805                 }
1806         }
1807         return NOTIFY_OK;
1808 }
1809
1810 static struct notifier_block __refdata cpufreq_cpu_notifier =
1811 {
1812     .notifier_call = cpufreq_cpu_callback,
1813 };
1814
1815 /*********************************************************************
1816  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1817  *********************************************************************/
1818
1819 /**
1820  * cpufreq_register_driver - register a CPU Frequency driver
1821  * @driver_data: A struct cpufreq_driver containing the values#
1822  * submitted by the CPU Frequency driver.
1823  *
1824  *   Registers a CPU Frequency driver to this core code. This code
1825  * returns zero on success, -EBUSY when another driver got here first
1826  * (and isn't unregistered in the meantime).
1827  *
1828  */
1829 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1830 {
1831         unsigned long flags;
1832         int ret;
1833
1834         if (!driver_data || !driver_data->verify || !driver_data->init ||
1835             ((!driver_data->setpolicy) && (!driver_data->target)))
1836                 return -EINVAL;
1837
1838         dprintk("trying to register driver %s\n", driver_data->name);
1839
1840         if (driver_data->setpolicy)
1841                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1842
1843         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1844         if (cpufreq_driver) {
1845                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1846                 return -EBUSY;
1847         }
1848         cpufreq_driver = driver_data;
1849         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1850
1851         ret = sysdev_driver_register(&cpu_sysdev_class,
1852                                         &cpufreq_sysdev_driver);
1853
1854         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1855                 int i;
1856                 ret = -ENODEV;
1857
1858                 /* check for at least one working CPU */
1859                 for (i = 0; i < nr_cpu_ids; i++)
1860                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1861                                 ret = 0;
1862                                 break;
1863                         }
1864
1865                 /* if all ->init() calls failed, unregister */
1866                 if (ret) {
1867                         dprintk("no CPU initialized for driver %s\n",
1868                                                         driver_data->name);
1869                         sysdev_driver_unregister(&cpu_sysdev_class,
1870                                                 &cpufreq_sysdev_driver);
1871
1872                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1873                         cpufreq_driver = NULL;
1874                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1875                 }
1876         }
1877
1878         if (!ret) {
1879                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1880                 dprintk("driver %s up and running\n", driver_data->name);
1881                 cpufreq_debug_enable_ratelimit();
1882         }
1883
1884         return ret;
1885 }
1886 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1887
1888
1889 /**
1890  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1891  *
1892  *    Unregister the current CPUFreq driver. Only call this if you have
1893  * the right to do so, i.e. if you have succeeded in initialising before!
1894  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1895  * currently not initialised.
1896  */
1897 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1898 {
1899         unsigned long flags;
1900
1901         cpufreq_debug_disable_ratelimit();
1902
1903         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1904                 cpufreq_debug_enable_ratelimit();
1905                 return -EINVAL;
1906         }
1907
1908         dprintk("unregistering driver %s\n", driver->name);
1909
1910         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1911         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1912
1913         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1914         cpufreq_driver = NULL;
1915         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1916
1917         return 0;
1918 }
1919 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1920
1921 static int __init cpufreq_core_init(void)
1922 {
1923         int cpu;
1924
1925         for_each_possible_cpu(cpu) {
1926                 per_cpu(policy_cpu, cpu) = -1;
1927                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1928         }
1929         return 0;
1930 }
1931
1932 core_initcall(cpufreq_core_init);