arch/s390/oprofile/hwsampler.c

   1 /**
   2  * arch/s390/oprofile/hwsampler.c
   3  *
   4  * Copyright IBM Corp. 2010
   5  * Author: Heinz Graalfs <graalfs@de.ibm.com>
   6  */
   7
   8 #include <linux/kernel_stat.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/smp.h>
  12 #include <linux/errno.h>
  13 #include <linux/workqueue.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/notifier.h>
  16 #include <linux/cpu.h>
  17 #include <linux/semaphore.h>
  18 #include <linux/oom.h>
  19 #include <linux/oprofile.h>
  20 #include <asm/cpu_mf.h>
  21 #include <asm/irq.h>
  22
  23 #include "hwsampler.h"
  24 #include "op_counter.h"
  25
  26 #define MAX_NUM_SDB 511
  27 #define MIN_NUM_SDB 1
  28
  29 #define ALERT_REQ_MASK   0x4000000000000000ul
  30 #define BUFFER_FULL_MASK 0x8000000000000000ul
  31
  32 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  33
  34 struct hws_execute_parms {
  35         void *buffer;
  36         signed int rc;
  37 };
  38
  39 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  40 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
  41
  42 static DEFINE_MUTEX(hws_sem);
  43 static DEFINE_MUTEX(hws_sem_oom);
  44
  45 static unsigned char hws_flush_all;
  46 static unsigned int hws_oom;
  47 static struct workqueue_struct *hws_wq;
  48
  49 static unsigned int hws_state;
  50 enum {
  51         HWS_INIT = 1,
  52         HWS_DEALLOCATED,
  53         HWS_STOPPED,
  54         HWS_STARTED,
  55         HWS_STOPPING };
  56
  57 /* set to 1 if called by kernel during memory allocation */
  58 static unsigned char oom_killer_was_active;
  59 /* size of SDBT and SDB as of allocate API */
  60 static unsigned long num_sdbt = 100;
  61 static unsigned long num_sdb = 511;
  62 /* sampling interval (machine cycles) */
  63 static unsigned long interval;
  64
  65 static unsigned long min_sampler_rate;
  66 static unsigned long max_sampler_rate;
  67
  68 static int ssctl(void *buffer)
  69 {
  70         int cc;
  71
  72         /* set in order to detect a program check */
  73         cc = 1;
  74
  75         asm volatile(
  76                 "0: .insn s,0xB2870000,0(%1)\n"
  77                 "1: ipm %0\n"
  78                 "   srl %0,28\n"
  79                 "2:\n"
  80                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  81                 : "+d" (cc), "+a" (buffer)
  82                 : "m" (*((struct hws_ssctl_request_block *)buffer))
  83                 : "cc", "memory");
  84
  85         return cc ? -EINVAL : 0 ;
  86 }
  87
  88 static int qsi(void *buffer)
  89 {
  90         int cc;
  91         cc = 1;
  92
  93         asm volatile(
  94                 "0: .insn s,0xB2860000,0(%1)\n"
  95                 "1: lhi %0,0\n"
  96                 "2:\n"
  97                 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
  98                 : "=d" (cc), "+a" (buffer)
  99                 : "m" (*((struct hws_qsi_info_block *)buffer))
 100                 : "cc", "memory");
 101
 102         return cc ? -EINVAL : 0;
 103 }
 104
 105 static void execute_qsi(void *parms)
 106 {
 107         struct hws_execute_parms *ep = parms;
 108
 109         ep->rc = qsi(ep->buffer);
 110 }
 111
 112 static void execute_ssctl(void *parms)
 113 {
 114         struct hws_execute_parms *ep = parms;
 115
 116         ep->rc = ssctl(ep->buffer);
 117 }
 118
 119 static int smp_ctl_ssctl_stop(int cpu)
 120 {
 121         int rc;
 122         struct hws_execute_parms ep;
 123         struct hws_cpu_buffer *cb;
 124
 125         cb = &per_cpu(sampler_cpu_buffer, cpu);
 126
 127         cb->ssctl.es = 0;
 128         cb->ssctl.cs = 0;
 129
 130         ep.buffer = &cb->ssctl;
 131         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 132         rc = ep.rc;
 133         if (rc) {
 134                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 135                 dump_stack();
 136         }
 137
 138         ep.buffer = &cb->qsi;
 139         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 140
 141         if (cb->qsi.es || cb->qsi.cs) {
 142                 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
 143                 dump_stack();
 144         }
 145
 146         return rc;
 147 }
 148
 149 static int smp_ctl_ssctl_deactivate(int cpu)
 150 {
 151         int rc;
 152         struct hws_execute_parms ep;
 153         struct hws_cpu_buffer *cb;
 154
 155         cb = &per_cpu(sampler_cpu_buffer, cpu);
 156
 157         cb->ssctl.es = 1;
 158         cb->ssctl.cs = 0;
 159
 160         ep.buffer = &cb->ssctl;
 161         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 162         rc = ep.rc;
 163         if (rc)
 164                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 165
 166         ep.buffer = &cb->qsi;
 167         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 168
 169         if (cb->qsi.cs)
 170                 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
 171
 172         return rc;
 173 }
 174
 175 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
 176 {
 177         int rc;
 178         struct hws_execute_parms ep;
 179         struct hws_cpu_buffer *cb;
 180
 181         cb = &per_cpu(sampler_cpu_buffer, cpu);
 182
 183         cb->ssctl.h = 1;
 184         cb->ssctl.tear = cb->first_sdbt;
 185         cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
 186         cb->ssctl.interval = interval;
 187         cb->ssctl.es = 1;
 188         cb->ssctl.cs = 1;
 189
 190         ep.buffer = &cb->ssctl;
 191         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 192         rc = ep.rc;
 193         if (rc)
 194                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 195
 196         ep.buffer = &cb->qsi;
 197         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 198         if (ep.rc)
 199                 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
 200
 201         return rc;
 202 }
 203
 204 static int smp_ctl_qsi(int cpu)
 205 {
 206         struct hws_execute_parms ep;
 207         struct hws_cpu_buffer *cb;
 208
 209         cb = &per_cpu(sampler_cpu_buffer, cpu);
 210
 211         ep.buffer = &cb->qsi;
 212         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 213
 214         return ep.rc;
 215 }
 216
 217 static inline unsigned long *trailer_entry_ptr(unsigned long v)
 218 {
 219         void *ret;
 220
 221         ret = (void *)v;
 222         ret += PAGE_SIZE;
 223         ret -= sizeof(struct hws_trailer_entry);
 224
 225         return (unsigned long *) ret;
 226 }
 227
 228 static void hws_ext_handler(struct ext_code ext_code,
 229                             unsigned int param32, unsigned long param64)
 230 {
 231         struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
 232
 233         if (!(param32 & CPU_MF_INT_SF_MASK))
 234                 return;
 235
 236         kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++;
 237         atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
 238
 239         if (hws_wq)
 240                 queue_work(hws_wq, &cb->worker);
 241 }
 242
 243 static void worker(struct work_struct *work);
 244
 245 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
 246                                 unsigned long *dear);
 247
 248 static void init_all_cpu_buffers(void)
 249 {
 250         int cpu;
 251         struct hws_cpu_buffer *cb;
 252
 253         for_each_online_cpu(cpu) {
 254                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 255                 memset(cb, 0, sizeof(struct hws_cpu_buffer));
 256         }
 257 }
 258
 259 static int is_link_entry(unsigned long *s)
 260 {
 261         return *s & 0x1ul ? 1 : 0;
 262 }
 263
 264 static unsigned long *get_next_sdbt(unsigned long *s)
 265 {
 266         return (unsigned long *) (*s & ~0x1ul);
 267 }
 268
 269 static int prepare_cpu_buffers(void)
 270 {
 271         int cpu;
 272         int rc;
 273         struct hws_cpu_buffer *cb;
 274
 275         rc = 0;
 276         for_each_online_cpu(cpu) {
 277                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 278                 atomic_set(&cb->ext_params, 0);
 279                 cb->worker_entry = 0;
 280                 cb->sample_overflow = 0;
 281                 cb->req_alert = 0;
 282                 cb->incorrect_sdbt_entry = 0;
 283                 cb->invalid_entry_address = 0;
 284                 cb->loss_of_sample_data = 0;
 285                 cb->sample_auth_change_alert = 0;
 286                 cb->finish = 0;
 287                 cb->oom = 0;
 288                 cb->stop_mode = 0;
 289         }
 290
 291         return rc;
 292 }
 293
 294 /*
 295  * allocate_sdbt() - allocate sampler memory
 296  * @cpu: the cpu for which sampler memory is allocated
 297  *
 298  * A 4K page is allocated for each requested SDBT.
 299  * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
 300  * Set ALERT_REQ mask in each SDBs trailer.
 301  * Returns zero if successful, <0 otherwise.
 302  */
 303 static int allocate_sdbt(int cpu)
 304 {
 305         int j, k, rc;
 306         unsigned long *sdbt;
 307         unsigned long  sdb;
 308         unsigned long *tail;
 309         unsigned long *trailer;
 310         struct hws_cpu_buffer *cb;
 311
 312         cb = &per_cpu(sampler_cpu_buffer, cpu);
 313
 314         if (cb->first_sdbt)
 315                 return -EINVAL;
 316
 317         sdbt = NULL;
 318         tail = sdbt;
 319
 320         for (j = 0; j < num_sdbt; j++) {
 321                 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 322
 323                 mutex_lock(&hws_sem_oom);
 324                 /* OOM killer might have been activated */
 325                 barrier();
 326                 if (oom_killer_was_active || !sdbt) {
 327                         if (sdbt)
 328                                 free_page((unsigned long)sdbt);
 329
 330                         goto allocate_sdbt_error;
 331                 }
 332                 if (cb->first_sdbt == 0)
 333                         cb->first_sdbt = (unsigned long)sdbt;
 334
 335                 /* link current page to tail of chain */
 336                 if (tail)
 337                         *tail = (unsigned long)(void *)sdbt + 1;
 338
 339                 mutex_unlock(&hws_sem_oom);
 340
 341                 for (k = 0; k < num_sdb; k++) {
 342                         /* get and set SDB page */
 343                         sdb = get_zeroed_page(GFP_KERNEL);
 344
 345                         mutex_lock(&hws_sem_oom);
 346                         /* OOM killer might have been activated */
 347                         barrier();
 348                         if (oom_killer_was_active || !sdb) {
 349                                 if (sdb)
 350                                         free_page(sdb);
 351
 352                                 goto allocate_sdbt_error;
 353                         }
 354                         *sdbt = sdb;
 355                         trailer = trailer_entry_ptr(*sdbt);
 356                         *trailer = ALERT_REQ_MASK;
 357                         sdbt++;
 358                         mutex_unlock(&hws_sem_oom);
 359                 }
 360                 tail = sdbt;
 361         }
 362         mutex_lock(&hws_sem_oom);
 363         if (oom_killer_was_active)
 364                 goto allocate_sdbt_error;
 365
 366         rc = 0;
 367         if (tail)
 368                 *tail = (unsigned long)
 369                         ((void *)cb->first_sdbt) + 1;
 370
 371 allocate_sdbt_exit:
 372         mutex_unlock(&hws_sem_oom);
 373         return rc;
 374
 375 allocate_sdbt_error:
 376         rc = -ENOMEM;
 377         goto allocate_sdbt_exit;
 378 }
 379
 380 /*
 381  * deallocate_sdbt() - deallocate all sampler memory
 382  *
 383  * For each online CPU all SDBT trees are deallocated.
 384  * Returns the number of freed pages.
 385  */
 386 static int deallocate_sdbt(void)
 387 {
 388         int cpu;
 389         int counter;
 390
 391         counter = 0;
 392
 393         for_each_online_cpu(cpu) {
 394                 unsigned long start;
 395                 unsigned long sdbt;
 396                 unsigned long *curr;
 397                 struct hws_cpu_buffer *cb;
 398
 399                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 400
 401                 if (!cb->first_sdbt)
 402                         continue;
 403
 404                 sdbt = cb->first_sdbt;
 405                 curr = (unsigned long *) sdbt;
 406                 start = sdbt;
 407
 408                 /* we'll free the SDBT after all SDBs are processed... */
 409                 while (1) {
 410                         if (!*curr || !sdbt)
 411                                 break;
 412
 413                         /* watch for link entry reset if found */
 414                         if (is_link_entry(curr)) {
 415                                 curr = get_next_sdbt(curr);
 416                                 if (sdbt)
 417                                         free_page(sdbt);
 418
 419                                 /* we are done if we reach the start */
 420                                 if ((unsigned long) curr == start)
 421                                         break;
 422                                 else
 423                                         sdbt = (unsigned long) curr;
 424                         } else {
 425                                 /* process SDB pointer */
 426                                 if (*curr) {
 427                                         free_page(*curr);
 428                                         curr++;
 429                                 }
 430                         }
 431                         counter++;
 432                 }
 433                 cb->first_sdbt = 0;
 434         }
 435         return counter;
 436 }
 437
 438 static int start_sampling(int cpu)
 439 {
 440         int rc;
 441         struct hws_cpu_buffer *cb;
 442
 443         cb = &per_cpu(sampler_cpu_buffer, cpu);
 444         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 445         if (rc) {
 446                 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
 447                 goto start_exit;
 448         }
 449
 450         rc = -EINVAL;
 451         if (!cb->qsi.es) {
 452                 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
 453                 goto start_exit;
 454         }
 455
 456         if (!cb->qsi.cs) {
 457                 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
 458                 goto start_exit;
 459         }
 460
 461         printk(KERN_INFO
 462                 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
 463                 cpu, interval);
 464
 465         rc = 0;
 466
 467 start_exit:
 468         return rc;
 469 }
 470
 471 static int stop_sampling(int cpu)
 472 {
 473         unsigned long v;
 474         int rc;
 475         struct hws_cpu_buffer *cb;
 476
 477         rc = smp_ctl_qsi(cpu);
 478         WARN_ON(rc);
 479
 480         cb = &per_cpu(sampler_cpu_buffer, cpu);
 481         if (!rc && !cb->qsi.es)
 482                 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
 483
 484         rc = smp_ctl_ssctl_stop(cpu);
 485         if (rc) {
 486                 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
 487                                 cpu, rc);
 488                 goto stop_exit;
 489         }
 490
 491         printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
 492
 493 stop_exit:
 494         v = cb->req_alert;
 495         if (v)
 496                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
 497                                 " count=%lu.\n", cpu, v);
 498
 499         v = cb->loss_of_sample_data;
 500         if (v)
 501                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
 502                                 " count=%lu.\n", cpu, v);
 503
 504         v = cb->invalid_entry_address;
 505         if (v)
 506                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
 507                                 " count=%lu.\n", cpu, v);
 508
 509         v = cb->incorrect_sdbt_entry;
 510         if (v)
 511                 printk(KERN_ERR
 512                                 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
 513                                 " count=%lu.\n", cpu, v);
 514
 515         v = cb->sample_auth_change_alert;
 516         if (v)
 517                 printk(KERN_ERR
 518                                 "hwsampler: CPU %d CPUMF Sample authorization change,"
 519                                 " count=%lu.\n", cpu, v);
 520
 521         return rc;
 522 }
 523
 524 static int check_hardware_prerequisites(void)
 525 {
 526         if (!test_facility(68))
 527                 return -EOPNOTSUPP;
 528         return 0;
 529 }
 530 /*
 531  * hws_oom_callback() - the OOM callback function
 532  *
 533  * In case the callback is invoked during memory allocation for the
 534  *  hw sampler, all obtained memory is deallocated and a flag is set
 535  *  so main sampler memory allocation can exit with a failure code.
 536  * In case the callback is invoked during sampling the hw sampler
 537  *  is deactivated for all CPUs.
 538  */
 539 static int hws_oom_callback(struct notifier_block *nfb,
 540         unsigned long dummy, void *parm)
 541 {
 542         unsigned long *freed;
 543         int cpu;
 544         struct hws_cpu_buffer *cb;
 545
 546         freed = parm;
 547
 548         mutex_lock(&hws_sem_oom);
 549
 550         if (hws_state == HWS_DEALLOCATED) {
 551                 /* during memory allocation */
 552                 if (oom_killer_was_active == 0) {
 553                         oom_killer_was_active = 1;
 554                         *freed += deallocate_sdbt();
 555                 }
 556         } else {
 557                 int i;
 558                 cpu = get_cpu();
 559                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 560
 561                 if (!cb->oom) {
 562                         for_each_online_cpu(i) {
 563                                 smp_ctl_ssctl_deactivate(i);
 564                                 cb->oom = 1;
 565                         }
 566                         cb->finish = 1;
 567
 568                         printk(KERN_INFO
 569                                 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
 570                                 cpu);
 571                 }
 572         }
 573
 574         mutex_unlock(&hws_sem_oom);
 575
 576         return NOTIFY_OK;
 577 }
 578
 579 static struct notifier_block hws_oom_notifier = {
 580         .notifier_call = hws_oom_callback
 581 };
 582
 583 static int hws_cpu_callback(struct notifier_block *nfb,
 584         unsigned long action, void *hcpu)
 585 {
 586         /* We do not have sampler space available for all possible CPUs.
 587            All CPUs should be online when hw sampling is activated. */
 588         return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
 589 }
 590
 591 static struct notifier_block hws_cpu_notifier = {
 592         .notifier_call = hws_cpu_callback
 593 };
 594
 595 /**
 596  * hwsampler_deactivate() - set hardware sampling temporarily inactive
 597  * @cpu:  specifies the CPU to be set inactive.
 598  *
 599  * Returns 0 on success, !0 on failure.
 600  */
 601 int hwsampler_deactivate(unsigned int cpu)
 602 {
 603         /*
 604          * Deactivate hw sampling temporarily and flush the buffer
 605          * by pushing all the pending samples to oprofile buffer.
 606          *
 607          * This function can be called under one of the following conditions:
 608          *     Memory unmap, task is exiting.
 609          */
 610         int rc;
 611         struct hws_cpu_buffer *cb;
 612
 613         rc = 0;
 614         mutex_lock(&hws_sem);
 615
 616         cb = &per_cpu(sampler_cpu_buffer, cpu);
 617         if (hws_state == HWS_STARTED) {
 618                 rc = smp_ctl_qsi(cpu);
 619                 WARN_ON(rc);
 620                 if (cb->qsi.cs) {
 621                         rc = smp_ctl_ssctl_deactivate(cpu);
 622                         if (rc) {
 623                                 printk(KERN_INFO
 624                                 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
 625                                 cb->finish = 1;
 626                                 hws_state = HWS_STOPPING;
 627                         } else  {
 628                                 hws_flush_all = 1;
 629                                 /* Add work to queue to read pending samples.*/
 630                                 queue_work_on(cpu, hws_wq, &cb->worker);
 631                         }
 632                 }
 633         }
 634         mutex_unlock(&hws_sem);
 635
 636         if (hws_wq)
 637                 flush_workqueue(hws_wq);
 638
 639         return rc;
 640 }
 641
 642 /**
 643  * hwsampler_activate() - activate/resume hardware sampling which was deactivated
 644  * @cpu:  specifies the CPU to be set active.
 645  *
 646  * Returns 0 on success, !0 on failure.
 647  */
 648 int hwsampler_activate(unsigned int cpu)
 649 {
 650         /*
 651          * Re-activate hw sampling. This should be called in pair with
 652          * hwsampler_deactivate().
 653          */
 654         int rc;
 655         struct hws_cpu_buffer *cb;
 656
 657         rc = 0;
 658         mutex_lock(&hws_sem);
 659
 660         cb = &per_cpu(sampler_cpu_buffer, cpu);
 661         if (hws_state == HWS_STARTED) {
 662                 rc = smp_ctl_qsi(cpu);
 663                 WARN_ON(rc);
 664                 if (!cb->qsi.cs) {
 665                         hws_flush_all = 0;
 666                         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 667                         if (rc) {
 668                                 printk(KERN_ERR
 669                                 "CPU %d, CPUMF activate sampling failed.\n",
 670                                          cpu);
 671                         }
 672                 }
 673         }
 674
 675         mutex_unlock(&hws_sem);
 676
 677         return rc;
 678 }
 679
 680 static int check_qsi_on_setup(void)
 681 {
 682         int rc;
 683         unsigned int cpu;
 684         struct hws_cpu_buffer *cb;
 685
 686         for_each_online_cpu(cpu) {
 687                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 688                 rc = smp_ctl_qsi(cpu);
 689                 WARN_ON(rc);
 690                 if (rc)
 691                         return -EOPNOTSUPP;
 692
 693                 if (!cb->qsi.as) {
 694                         printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
 695                         return -EINVAL;
 696                 }
 697
 698                 if (cb->qsi.es) {
 699                         printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
 700                         rc = smp_ctl_ssctl_stop(cpu);
 701                         if (rc)
 702                                 return -EINVAL;
 703
 704                         printk(KERN_INFO
 705                                 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
 706                 }
 707         }
 708         return 0;
 709 }
 710
 711 static int check_qsi_on_start(void)
 712 {
 713         unsigned int cpu;
 714         int rc;
 715         struct hws_cpu_buffer *cb;
 716
 717         for_each_online_cpu(cpu) {
 718                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 719                 rc = smp_ctl_qsi(cpu);
 720                 WARN_ON(rc);
 721
 722                 if (!cb->qsi.as)
 723                         return -EINVAL;
 724
 725                 if (cb->qsi.es)
 726                         return -EINVAL;
 727
 728                 if (cb->qsi.cs)
 729                         return -EINVAL;
 730         }
 731         return 0;
 732 }
 733
 734 static void worker_on_start(unsigned int cpu)
 735 {
 736         struct hws_cpu_buffer *cb;
 737
 738         cb = &per_cpu(sampler_cpu_buffer, cpu);
 739         cb->worker_entry = cb->first_sdbt;
 740 }
 741
 742 static int worker_check_error(unsigned int cpu, int ext_params)
 743 {
 744         int rc;
 745         unsigned long *sdbt;
 746         struct hws_cpu_buffer *cb;
 747
 748         rc = 0;
 749         cb = &per_cpu(sampler_cpu_buffer, cpu);
 750         sdbt = (unsigned long *) cb->worker_entry;
 751
 752         if (!sdbt || !*sdbt)
 753                 return -EINVAL;
 754
 755         if (ext_params & CPU_MF_INT_SF_PRA)
 756                 cb->req_alert++;
 757
 758         if (ext_params & CPU_MF_INT_SF_LSDA)
 759                 cb->loss_of_sample_data++;
 760
 761         if (ext_params & CPU_MF_INT_SF_IAE) {
 762                 cb->invalid_entry_address++;
 763                 rc = -EINVAL;
 764         }
 765
 766         if (ext_params & CPU_MF_INT_SF_ISE) {
 767                 cb->incorrect_sdbt_entry++;
 768                 rc = -EINVAL;
 769         }
 770
 771         if (ext_params & CPU_MF_INT_SF_SACA) {
 772                 cb->sample_auth_change_alert++;
 773                 rc = -EINVAL;
 774         }
 775
 776         return rc;
 777 }
 778
 779 static void worker_on_finish(unsigned int cpu)
 780 {
 781         int rc, i;
 782         struct hws_cpu_buffer *cb;
 783
 784         cb = &per_cpu(sampler_cpu_buffer, cpu);
 785
 786         if (cb->finish) {
 787                 rc = smp_ctl_qsi(cpu);
 788                 WARN_ON(rc);
 789                 if (cb->qsi.es) {
 790                         printk(KERN_INFO
 791                                 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
 792                                 cpu);
 793                         rc = smp_ctl_ssctl_stop(cpu);
 794                         if (rc)
 795                                 printk(KERN_INFO
 796                                         "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
 797                                         cpu);
 798
 799                         for_each_online_cpu(i) {
 800                                 if (i == cpu)
 801                                         continue;
 802                                 if (!cb->finish) {
 803                                         cb->finish = 1;
 804                                         queue_work_on(i, hws_wq,
 805                                                 &cb->worker);
 806                                 }
 807                         }
 808                 }
 809         }
 810 }
 811
 812 static void worker_on_interrupt(unsigned int cpu)
 813 {
 814         unsigned long *sdbt;
 815         unsigned char done;
 816         struct hws_cpu_buffer *cb;
 817
 818         cb = &per_cpu(sampler_cpu_buffer, cpu);
 819
 820         sdbt = (unsigned long *) cb->worker_entry;
 821
 822         done = 0;
 823         /* do not proceed if stop was entered,
 824          * forget the buffers not yet processed */
 825         while (!done && !cb->stop_mode) {
 826                 unsigned long *trailer;
 827                 struct hws_trailer_entry *te;
 828                 unsigned long *dear = 0;
 829
 830                 trailer = trailer_entry_ptr(*sdbt);
 831                 /* leave loop if no more work to do */
 832                 if (!(*trailer & BUFFER_FULL_MASK)) {
 833                         done = 1;
 834                         if (!hws_flush_all)
 835                                 continue;
 836                 }
 837
 838                 te = (struct hws_trailer_entry *)trailer;
 839                 cb->sample_overflow += te->overflow;
 840
 841                 add_samples_to_oprofile(cpu, sdbt, dear);
 842
 843                 /* reset trailer */
 844                 xchg((unsigned char *) te, 0x40);
 845
 846                 /* advance to next sdb slot in current sdbt */
 847                 sdbt++;
 848                 /* in case link bit is set use address w/o link bit */
 849                 if (is_link_entry(sdbt))
 850                         sdbt = get_next_sdbt(sdbt);
 851
 852                 cb->worker_entry = (unsigned long)sdbt;
 853         }
 854 }
 855
 856 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
 857                 unsigned long *dear)
 858 {
 859         struct hws_data_entry *sample_data_ptr;
 860         unsigned long *trailer;
 861
 862         trailer = trailer_entry_ptr(*sdbt);
 863         if (dear) {
 864                 if (dear > trailer)
 865                         return;
 866                 trailer = dear;
 867         }
 868
 869         sample_data_ptr = (struct hws_data_entry *)(*sdbt);
 870
 871         while ((unsigned long *)sample_data_ptr < trailer) {
 872                 struct pt_regs *regs = NULL;
 873                 struct task_struct *tsk = NULL;
 874
 875                 /*
 876                  * Check sampling mode, 1 indicates basic (=customer) sampling
 877                  * mode.
 878                  */
 879                 if (sample_data_ptr->def != 1) {
 880                         /* sample slot is not yet written */
 881                         break;
 882                 } else {
 883                         /* make sure we don't use it twice,
 884                          * the next time the sampler will set it again */
 885                         sample_data_ptr->def = 0;
 886                 }
 887
 888                 /* Get pt_regs. */
 889                 if (sample_data_ptr->P == 1) {
 890                         /* userspace sample */
 891                         unsigned int pid = sample_data_ptr->prim_asn;
 892                         if (!counter_config.user)
 893                                 goto skip_sample;
 894                         rcu_read_lock();
 895                         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 896                         if (tsk)
 897                                 regs = task_pt_regs(tsk);
 898                         rcu_read_unlock();
 899                 } else {
 900                         /* kernelspace sample */
 901                         if (!counter_config.kernel)
 902                                 goto skip_sample;
 903                         regs = task_pt_regs(current);
 904                 }
 905
 906                 mutex_lock(&hws_sem);
 907                 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
 908                                 !sample_data_ptr->P, tsk);
 909                 mutex_unlock(&hws_sem);
 910         skip_sample:
 911                 sample_data_ptr++;
 912         }
 913 }
 914
 915 static void worker(struct work_struct *work)
 916 {
 917         unsigned int cpu;
 918         int ext_params;
 919         struct hws_cpu_buffer *cb;
 920
 921         cb = container_of(work, struct hws_cpu_buffer, worker);
 922         cpu = smp_processor_id();
 923         ext_params = atomic_xchg(&cb->ext_params, 0);
 924
 925         if (!cb->worker_entry)
 926                 worker_on_start(cpu);
 927
 928         if (worker_check_error(cpu, ext_params))
 929                 return;
 930
 931         if (!cb->finish)
 932                 worker_on_interrupt(cpu);
 933
 934         if (cb->finish)
 935                 worker_on_finish(cpu);
 936 }
 937
 938 /**
 939  * hwsampler_allocate() - allocate memory for the hardware sampler
 940  * @sdbt:  number of SDBTs per online CPU (must be > 0)
 941  * @sdb:   number of SDBs per SDBT (minimum 1, maximum 511)
 942  *
 943  * Returns 0 on success, !0 on failure.
 944  */
 945 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
 946 {
 947         int cpu, rc;
 948         mutex_lock(&hws_sem);
 949
 950         rc = -EINVAL;
 951         if (hws_state != HWS_DEALLOCATED)
 952                 goto allocate_exit;
 953
 954         if (sdbt < 1)
 955                 goto allocate_exit;
 956
 957         if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
 958                 goto allocate_exit;
 959
 960         num_sdbt = sdbt;
 961         num_sdb = sdb;
 962
 963         oom_killer_was_active = 0;
 964         register_oom_notifier(&hws_oom_notifier);
 965
 966         for_each_online_cpu(cpu) {
 967                 if (allocate_sdbt(cpu)) {
 968                         unregister_oom_notifier(&hws_oom_notifier);
 969                         goto allocate_error;
 970                 }
 971         }
 972         unregister_oom_notifier(&hws_oom_notifier);
 973         if (oom_killer_was_active)
 974                 goto allocate_error;
 975
 976         hws_state = HWS_STOPPED;
 977         rc = 0;
 978
 979 allocate_exit:
 980         mutex_unlock(&hws_sem);
 981         return rc;
 982
 983 allocate_error:
 984         rc = -ENOMEM;
 985         printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
 986         goto allocate_exit;
 987 }
 988
 989 /**
 990  * hwsampler_deallocate() - deallocate hardware sampler memory
 991  *
 992  * Returns 0 on success, !0 on failure.
 993  */
 994 int hwsampler_deallocate(void)
 995 {
 996         int rc;
 997
 998         mutex_lock(&hws_sem);
 999
1000         rc = -EINVAL;
1001         if (hws_state != HWS_STOPPED)
1002                 goto deallocate_exit;
1003
1004         measurement_alert_subclass_unregister();
1005         deallocate_sdbt();
1006
1007         hws_state = HWS_DEALLOCATED;
1008         rc = 0;
1009
1010 deallocate_exit:
1011         mutex_unlock(&hws_sem);
1012
1013         return rc;
1014 }
1015
1016 unsigned long hwsampler_query_min_interval(void)
1017 {
1018         return min_sampler_rate;
1019 }
1020
1021 unsigned long hwsampler_query_max_interval(void)
1022 {
1023         return max_sampler_rate;
1024 }
1025
1026 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1027 {
1028         struct hws_cpu_buffer *cb;
1029
1030         cb = &per_cpu(sampler_cpu_buffer, cpu);
1031
1032         return cb->sample_overflow;
1033 }
1034
1035 int hwsampler_setup(void)
1036 {
1037         int rc;
1038         int cpu;
1039         struct hws_cpu_buffer *cb;
1040
1041         mutex_lock(&hws_sem);
1042
1043         rc = -EINVAL;
1044         if (hws_state)
1045                 goto setup_exit;
1046
1047         hws_state = HWS_INIT;
1048
1049         init_all_cpu_buffers();
1050
1051         rc = check_hardware_prerequisites();
1052         if (rc)
1053                 goto setup_exit;
1054
1055         rc = check_qsi_on_setup();
1056         if (rc)
1057                 goto setup_exit;
1058
1059         rc = -EINVAL;
1060         hws_wq = create_workqueue("hwsampler");
1061         if (!hws_wq)
1062                 goto setup_exit;
1063
1064         register_cpu_notifier(&hws_cpu_notifier);
1065
1066         for_each_online_cpu(cpu) {
1067                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1068                 INIT_WORK(&cb->worker, worker);
1069                 rc = smp_ctl_qsi(cpu);
1070                 WARN_ON(rc);
1071                 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1072                         if (min_sampler_rate) {
1073                                 printk(KERN_WARNING
1074                                         "hwsampler: different min sampler rate values.\n");
1075                                 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1076                                         min_sampler_rate =
1077                                                 cb->qsi.min_sampl_rate;
1078                         } else
1079                                 min_sampler_rate = cb->qsi.min_sampl_rate;
1080                 }
1081                 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1082                         if (max_sampler_rate) {
1083                                 printk(KERN_WARNING
1084                                         "hwsampler: different max sampler rate values.\n");
1085                                 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1086                                         max_sampler_rate =
1087                                                 cb->qsi.max_sampl_rate;
1088                         } else
1089                                 max_sampler_rate = cb->qsi.max_sampl_rate;
1090                 }
1091         }
1092         register_external_interrupt(0x1407, hws_ext_handler);
1093
1094         hws_state = HWS_DEALLOCATED;
1095         rc = 0;
1096
1097 setup_exit:
1098         mutex_unlock(&hws_sem);
1099         return rc;
1100 }
1101
1102 int hwsampler_shutdown(void)
1103 {
1104         int rc;
1105
1106         mutex_lock(&hws_sem);
1107
1108         rc = -EINVAL;
1109         if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1110                 mutex_unlock(&hws_sem);
1111
1112                 if (hws_wq)
1113                         flush_workqueue(hws_wq);
1114
1115                 mutex_lock(&hws_sem);
1116
1117                 if (hws_state == HWS_STOPPED) {
1118                         measurement_alert_subclass_unregister();
1119                         deallocate_sdbt();
1120                 }
1121                 if (hws_wq) {
1122                         destroy_workqueue(hws_wq);
1123                         hws_wq = NULL;
1124                 }
1125
1126                 unregister_external_interrupt(0x1407, hws_ext_handler);
1127                 hws_state = HWS_INIT;
1128                 rc = 0;
1129         }
1130         mutex_unlock(&hws_sem);
1131
1132         unregister_cpu_notifier(&hws_cpu_notifier);
1133
1134         return rc;
1135 }
1136
1137 /**
1138  * hwsampler_start_all() - start hardware sampling on all online CPUs
1139  * @rate:  specifies the used interval when samples are taken
1140  *
1141  * Returns 0 on success, !0 on failure.
1142  */
1143 int hwsampler_start_all(unsigned long rate)
1144 {
1145         int rc, cpu;
1146
1147         mutex_lock(&hws_sem);
1148
1149         hws_oom = 0;
1150
1151         rc = -EINVAL;
1152         if (hws_state != HWS_STOPPED)
1153                 goto start_all_exit;
1154
1155         interval = rate;
1156
1157         /* fail if rate is not valid */
1158         if (interval < min_sampler_rate || interval > max_sampler_rate)
1159                 goto start_all_exit;
1160
1161         rc = check_qsi_on_start();
1162         if (rc)
1163                 goto start_all_exit;
1164
1165         rc = prepare_cpu_buffers();
1166         if (rc)
1167                 goto start_all_exit;
1168
1169         for_each_online_cpu(cpu) {
1170                 rc = start_sampling(cpu);
1171                 if (rc)
1172                         break;
1173         }
1174         if (rc) {
1175                 for_each_online_cpu(cpu) {
1176                         stop_sampling(cpu);
1177                 }
1178                 goto start_all_exit;
1179         }
1180         hws_state = HWS_STARTED;
1181         rc = 0;
1182
1183 start_all_exit:
1184         mutex_unlock(&hws_sem);
1185
1186         if (rc)
1187                 return rc;
1188
1189         register_oom_notifier(&hws_oom_notifier);
1190         hws_oom = 1;
1191         hws_flush_all = 0;
1192         /* now let them in, 1407 CPUMF external interrupts */
1193         measurement_alert_subclass_register();
1194
1195         return 0;
1196 }
1197
1198 /**
1199  * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1200  *
1201  * Returns 0 on success, !0 on failure.
1202  */
1203 int hwsampler_stop_all(void)
1204 {
1205         int tmp_rc, rc, cpu;
1206         struct hws_cpu_buffer *cb;
1207
1208         mutex_lock(&hws_sem);
1209
1210         rc = 0;
1211         if (hws_state == HWS_INIT) {
1212                 mutex_unlock(&hws_sem);
1213                 return rc;
1214         }
1215         hws_state = HWS_STOPPING;
1216         mutex_unlock(&hws_sem);
1217
1218         for_each_online_cpu(cpu) {
1219                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1220                 cb->stop_mode = 1;
1221                 tmp_rc = stop_sampling(cpu);
1222                 if (tmp_rc)
1223                         rc = tmp_rc;
1224         }
1225
1226         if (hws_wq)
1227                 flush_workqueue(hws_wq);
1228
1229         mutex_lock(&hws_sem);
1230         if (hws_oom) {
1231                 unregister_oom_notifier(&hws_oom_notifier);
1232                 hws_oom = 0;
1233         }
1234         hws_state = HWS_STOPPED;
1235         mutex_unlock(&hws_sem);
1236
1237         return rc;
1238 }