kernel/kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  *
  21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
  22  *              Probes initial implementation (includes suggestions from
  23  *              Rusty Russell).
  24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
  25  *              hlists and exceptions notifier as suggested by Andi Kleen.
  26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
  27  *              interface to access function arguments.
  28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
  29  *              exceptions notifier to be first on the priority list.
  30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
  31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
  32  *              <prasanna@in.ibm.com> added function-return probes.
  33  */
  34 #include <linux/kprobes.h>
  35 #include <linux/hash.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/stddef.h>
  39 #include <linux/module.h>
  40 #include <linux/moduleloader.h>
  41 #include <linux/kallsyms.h>
  42 #include <linux/freezer.h>
  43 #include <linux/seq_file.h>
  44 #include <linux/debugfs.h>
  45 #include <linux/kdebug.h>
  46 #include <linux/memory.h>
  47
  48 #include <asm-generic/sections.h>
  49 #include <asm/cacheflush.h>
  50 #include <asm/errno.h>
  51 #include <asm/uaccess.h>
  52
  53 #define KPROBE_HASH_BITS 6
  54 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
  55
  56
  57 /*
  58  * Some oddball architectures like 64bit powerpc have function descriptors
  59  * so this must be overridable.
  60  */
  61 #ifndef kprobe_lookup_name
  62 #define kprobe_lookup_name(name, addr) \
  63         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
  64 #endif
  65
  66 static int kprobes_initialized;
  67 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  68 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  69
  70 /* NOTE: change this value only with kprobe_mutex held */
  71 static bool kprobes_all_disarmed;
  72
  73 static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
  74 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  75 static struct {
  76         spinlock_t lock ____cacheline_aligned_in_smp;
  77 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
  78
  79 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
  80 {
  81         return &(kretprobe_table_locks[hash].lock);
  82 }
  83
  84 /*
  85  * Normally, functions that we'd want to prohibit kprobes in, are marked
  86  * __kprobes. But, there are cases where such functions already belong to
  87  * a different section (__sched for preempt_schedule)
  88  *
  89  * For such cases, we now have a blacklist
  90  */
  91 static struct kprobe_blackpoint kprobe_blacklist[] = {
  92         {"preempt_schedule",},
  93         {"native_get_debugreg",},
  94         {NULL}    /* Terminator */
  95 };
  96
  97 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
  98 /*
  99  * kprobe->ainsn.insn points to the copy of the instruction to be
 100  * single-stepped. x86_64, POWER4 and above have no-exec support and
 101  * stepping on the instruction on a vmalloced/kmalloced/data page
 102  * is a recipe for disaster
 103  */
 104 #define INSNS_PER_PAGE  (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
 105
 106 struct kprobe_insn_page {
 107         struct list_head list;
 108         kprobe_opcode_t *insns;         /* Page of instruction slots */
 109         char slot_used[INSNS_PER_PAGE];
 110         int nused;
 111         int ngarbage;
 112 };
 113
 114 enum kprobe_slot_state {
 115         SLOT_CLEAN = 0,
 116         SLOT_DIRTY = 1,
 117         SLOT_USED = 2,
 118 };
 119
 120 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_pages */
 121 static LIST_HEAD(kprobe_insn_pages);
 122 static int kprobe_garbage_slots;
 123 static int collect_garbage_slots(void);
 124
 125 static int __kprobes check_safety(void)
 126 {
 127         int ret = 0;
 128 #if defined(CONFIG_PREEMPT) && defined(CONFIG_FREEZER)
 129         ret = freeze_processes();
 130         if (ret == 0) {
 131                 struct task_struct *p, *q;
 132                 do_each_thread(p, q) {
 133                         if (p != current && p->state == TASK_RUNNING &&
 134                             p->pid != 0) {
 135                                 printk("Check failed: %s is running\n",p->comm);
 136                                 ret = -1;
 137                                 goto loop_end;
 138                         }
 139                 } while_each_thread(p, q);
 140         }
 141 loop_end:
 142         thaw_processes();
 143 #else
 144         synchronize_sched();
 145 #endif
 146         return ret;
 147 }
 148
 149 /**
 150  * __get_insn_slot() - Find a slot on an executable page for an instruction.
 151  * We allocate an executable page if there's no room on existing ones.
 152  */
 153 static kprobe_opcode_t __kprobes *__get_insn_slot(void)
 154 {
 155         struct kprobe_insn_page *kip;
 156
 157  retry:
 158         list_for_each_entry(kip, &kprobe_insn_pages, list) {
 159                 if (kip->nused < INSNS_PER_PAGE) {
 160                         int i;
 161                         for (i = 0; i < INSNS_PER_PAGE; i++) {
 162                                 if (kip->slot_used[i] == SLOT_CLEAN) {
 163                                         kip->slot_used[i] = SLOT_USED;
 164                                         kip->nused++;
 165                                         return kip->insns + (i * MAX_INSN_SIZE);
 166                                 }
 167                         }
 168                         /* Surprise!  No unused slots.  Fix kip->nused. */
 169                         kip->nused = INSNS_PER_PAGE;
 170                 }
 171         }
 172
 173         /* If there are any garbage slots, collect it and try again. */
 174         if (kprobe_garbage_slots && collect_garbage_slots() == 0) {
 175                 goto retry;
 176         }
 177         /* All out of space.  Need to allocate a new page. Use slot 0. */
 178         kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
 179         if (!kip)
 180                 return NULL;
 181
 182         /*
 183          * Use module_alloc so this page is within +/- 2GB of where the
 184          * kernel image and loaded module images reside. This is required
 185          * so x86_64 can correctly handle the %rip-relative fixups.
 186          */
 187         kip->insns = module_alloc(PAGE_SIZE);
 188         if (!kip->insns) {
 189                 kfree(kip);
 190                 return NULL;
 191         }
 192         INIT_LIST_HEAD(&kip->list);
 193         list_add(&kip->list, &kprobe_insn_pages);
 194         memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
 195         kip->slot_used[0] = SLOT_USED;
 196         kip->nused = 1;
 197         kip->ngarbage = 0;
 198         return kip->insns;
 199 }
 200
 201 kprobe_opcode_t __kprobes *get_insn_slot(void)
 202 {
 203         kprobe_opcode_t *ret;
 204         mutex_lock(&kprobe_insn_mutex);
 205         ret = __get_insn_slot();
 206         mutex_unlock(&kprobe_insn_mutex);
 207         return ret;
 208 }
 209
 210 /* Return 1 if all garbages are collected, otherwise 0. */
 211 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
 212 {
 213         kip->slot_used[idx] = SLOT_CLEAN;
 214         kip->nused--;
 215         if (kip->nused == 0) {
 216                 /*
 217                  * Page is no longer in use.  Free it unless
 218                  * it's the last one.  We keep the last one
 219                  * so as not to have to set it up again the
 220                  * next time somebody inserts a probe.
 221                  */
 222                 if (!list_is_singular(&kprobe_insn_pages)) {
 223                         list_del(&kip->list);
 224                         module_free(NULL, kip->insns);
 225                         kfree(kip);
 226                 }
 227                 return 1;
 228         }
 229         return 0;
 230 }
 231
 232 static int __kprobes collect_garbage_slots(void)
 233 {
 234         struct kprobe_insn_page *kip, *next;
 235
 236         /* Ensure no-one is preepmted on the garbages */
 237         if (check_safety())
 238                 return -EAGAIN;
 239
 240         list_for_each_entry_safe(kip, next, &kprobe_insn_pages, list) {
 241                 int i;
 242                 if (kip->ngarbage == 0)
 243                         continue;
 244                 kip->ngarbage = 0;      /* we will collect all garbages */
 245                 for (i = 0; i < INSNS_PER_PAGE; i++) {
 246                         if (kip->slot_used[i] == SLOT_DIRTY &&
 247                             collect_one_slot(kip, i))
 248                                 break;
 249                 }
 250         }
 251         kprobe_garbage_slots = 0;
 252         return 0;
 253 }
 254
 255 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
 256 {
 257         struct kprobe_insn_page *kip;
 258
 259         mutex_lock(&kprobe_insn_mutex);
 260         list_for_each_entry(kip, &kprobe_insn_pages, list) {
 261                 if (kip->insns <= slot &&
 262                     slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
 263                         int i = (slot - kip->insns) / MAX_INSN_SIZE;
 264                         if (dirty) {
 265                                 kip->slot_used[i] = SLOT_DIRTY;
 266                                 kip->ngarbage++;
 267                         } else
 268                                 collect_one_slot(kip, i);
 269                         break;
 270                 }
 271         }
 272
 273         if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)
 274                 collect_garbage_slots();
 275
 276         mutex_unlock(&kprobe_insn_mutex);
 277 }
 278 #endif
 279
 280 /* We have preemption disabled.. so it is safe to use __ versions */
 281 static inline void set_kprobe_instance(struct kprobe *kp)
 282 {
 283         __get_cpu_var(kprobe_instance) = kp;
 284 }
 285
 286 static inline void reset_kprobe_instance(void)
 287 {
 288         __get_cpu_var(kprobe_instance) = NULL;
 289 }
 290
 291 /*
 292  * This routine is called either:
 293  *      - under the kprobe_mutex - during kprobe_[un]register()
 294  *                              OR
 295  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 296  */
 297 struct kprobe __kprobes *get_kprobe(void *addr)
 298 {
 299         struct hlist_head *head;
 300         struct hlist_node *node;
 301         struct kprobe *p;
 302
 303         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
 304         hlist_for_each_entry_rcu(p, node, head, hlist) {
 305                 if (p->addr == addr)
 306                         return p;
 307         }
 308         return NULL;
 309 }
 310
 311 /* Arm a kprobe with text_mutex */
 312 static void __kprobes arm_kprobe(struct kprobe *kp)
 313 {
 314         mutex_lock(&text_mutex);
 315         arch_arm_kprobe(kp);
 316         mutex_unlock(&text_mutex);
 317 }
 318
 319 /* Disarm a kprobe with text_mutex */
 320 static void __kprobes disarm_kprobe(struct kprobe *kp)
 321 {
 322         mutex_lock(&text_mutex);
 323         arch_disarm_kprobe(kp);
 324         mutex_unlock(&text_mutex);
 325 }
 326
 327 /*
 328  * Aggregate handlers for multiple kprobes support - these handlers
 329  * take care of invoking the individual kprobe handlers on p->list
 330  */
 331 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 332 {
 333         struct kprobe *kp;
 334
 335         list_for_each_entry_rcu(kp, &p->list, list) {
 336                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
 337                         set_kprobe_instance(kp);
 338                         if (kp->pre_handler(kp, regs))
 339                                 return 1;
 340                 }
 341                 reset_kprobe_instance();
 342         }
 343         return 0;
 344 }
 345
 346 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
 347                                         unsigned long flags)
 348 {
 349         struct kprobe *kp;
 350
 351         list_for_each_entry_rcu(kp, &p->list, list) {
 352                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
 353                         set_kprobe_instance(kp);
 354                         kp->post_handler(kp, regs, flags);
 355                         reset_kprobe_instance();
 356                 }
 357         }
 358 }
 359
 360 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
 361                                         int trapnr)
 362 {
 363         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 364
 365         /*
 366          * if we faulted "during" the execution of a user specified
 367          * probe handler, invoke just that probe's fault handler
 368          */
 369         if (cur && cur->fault_handler) {
 370                 if (cur->fault_handler(cur, regs, trapnr))
 371                         return 1;
 372         }
 373         return 0;
 374 }
 375
 376 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 377 {
 378         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 379         int ret = 0;
 380
 381         if (cur && cur->break_handler) {
 382                 if (cur->break_handler(cur, regs))
 383                         ret = 1;
 384         }
 385         reset_kprobe_instance();
 386         return ret;
 387 }
 388
 389 /* Walks the list and increments nmissed count for multiprobe case */
 390 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
 391 {
 392         struct kprobe *kp;
 393         if (p->pre_handler != aggr_pre_handler) {
 394                 p->nmissed++;
 395         } else {
 396                 list_for_each_entry_rcu(kp, &p->list, list)
 397                         kp->nmissed++;
 398         }
 399         return;
 400 }
 401
 402 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
 403                                 struct hlist_head *head)
 404 {
 405         struct kretprobe *rp = ri->rp;
 406
 407         /* remove rp inst off the rprobe_inst_table */
 408         hlist_del(&ri->hlist);
 409         INIT_HLIST_NODE(&ri->hlist);
 410         if (likely(rp)) {
 411                 spin_lock(&rp->lock);
 412                 hlist_add_head(&ri->hlist, &rp->free_instances);
 413                 spin_unlock(&rp->lock);
 414         } else
 415                 /* Unregistering */
 416                 hlist_add_head(&ri->hlist, head);
 417 }
 418
 419 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
 420                          struct hlist_head **head, unsigned long *flags)
 421 {
 422         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 423         spinlock_t *hlist_lock;
 424
 425         *head = &kretprobe_inst_table[hash];
 426         hlist_lock = kretprobe_table_lock_ptr(hash);
 427         spin_lock_irqsave(hlist_lock, *flags);
 428 }
 429
 430 static void __kprobes kretprobe_table_lock(unsigned long hash,
 431         unsigned long *flags)
 432 {
 433         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 434         spin_lock_irqsave(hlist_lock, *flags);
 435 }
 436
 437 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
 438         unsigned long *flags)
 439 {
 440         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
 441         spinlock_t *hlist_lock;
 442
 443         hlist_lock = kretprobe_table_lock_ptr(hash);
 444         spin_unlock_irqrestore(hlist_lock, *flags);
 445 }
 446
 447 void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
 448 {
 449         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
 450         spin_unlock_irqrestore(hlist_lock, *flags);
 451 }
 452
 453 /*
 454  * This function is called from finish_task_switch when task tk becomes dead,
 455  * so that we can recycle any function-return probe instances associated
 456  * with this task. These left over instances represent probed functions
 457  * that have been called but will never return.
 458  */
 459 void __kprobes kprobe_flush_task(struct task_struct *tk)
 460 {
 461         struct kretprobe_instance *ri;
 462         struct hlist_head *head, empty_rp;
 463         struct hlist_node *node, *tmp;
 464         unsigned long hash, flags = 0;
 465
 466         if (unlikely(!kprobes_initialized))
 467                 /* Early boot.  kretprobe_table_locks not yet initialized. */
 468                 return;
 469
 470         hash = hash_ptr(tk, KPROBE_HASH_BITS);
 471         head = &kretprobe_inst_table[hash];
 472         kretprobe_table_lock(hash, &flags);
 473         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 474                 if (ri->task == tk)
 475                         recycle_rp_inst(ri, &empty_rp);
 476         }
 477         kretprobe_table_unlock(hash, &flags);
 478         INIT_HLIST_HEAD(&empty_rp);
 479         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
 480                 hlist_del(&ri->hlist);
 481                 kfree(ri);
 482         }
 483 }
 484
 485 static inline void free_rp_inst(struct kretprobe *rp)
 486 {
 487         struct kretprobe_instance *ri;
 488         struct hlist_node *pos, *next;
 489
 490         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
 491                 hlist_del(&ri->hlist);
 492                 kfree(ri);
 493         }
 494 }
 495
 496 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
 497 {
 498         unsigned long flags, hash;
 499         struct kretprobe_instance *ri;
 500         struct hlist_node *pos, *next;
 501         struct hlist_head *head;
 502
 503         /* No race here */
 504         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
 505                 kretprobe_table_lock(hash, &flags);
 506                 head = &kretprobe_inst_table[hash];
 507                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
 508                         if (ri->rp == rp)
 509                                 ri->rp = NULL;
 510                 }
 511                 kretprobe_table_unlock(hash, &flags);
 512         }
 513         free_rp_inst(rp);
 514 }
 515
 516 /*
 517  * Keep all fields in the kprobe consistent
 518  */
 519 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 520 {
 521         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 522         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 523 }
 524
 525 /*
 526 * Add the new probe to ap->list. Fail if this is the
 527 * second jprobe at the address - two jprobes can't coexist
 528 */
 529 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
 530 {
 531         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
 532         if (p->break_handler) {
 533                 if (ap->break_handler)
 534                         return -EEXIST;
 535                 list_add_tail_rcu(&p->list, &ap->list);
 536                 ap->break_handler = aggr_break_handler;
 537         } else
 538                 list_add_rcu(&p->list, &ap->list);
 539         if (p->post_handler && !ap->post_handler)
 540                 ap->post_handler = aggr_post_handler;
 541
 542         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
 543                 ap->flags &= ~KPROBE_FLAG_DISABLED;
 544                 if (!kprobes_all_disarmed)
 545                         /* Arm the breakpoint again. */
 546                         arm_kprobe(ap);
 547         }
 548         return 0;
 549 }
 550
 551 /*
 552  * Fill in the required fields of the "manager kprobe". Replace the
 553  * earlier kprobe in the hlist with the manager kprobe
 554  */
 555 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 556 {
 557         copy_kprobe(p, ap);
 558         flush_insn_slot(ap);
 559         ap->addr = p->addr;
 560         ap->flags = p->flags;
 561         ap->pre_handler = aggr_pre_handler;
 562         ap->fault_handler = aggr_fault_handler;
 563         /* We don't care the kprobe which has gone. */
 564         if (p->post_handler && !kprobe_gone(p))
 565                 ap->post_handler = aggr_post_handler;
 566         if (p->break_handler && !kprobe_gone(p))
 567                 ap->break_handler = aggr_break_handler;
 568
 569         INIT_LIST_HEAD(&ap->list);
 570         list_add_rcu(&p->list, &ap->list);
 571
 572         hlist_replace_rcu(&p->hlist, &ap->hlist);
 573 }
 574
 575 /*
 576  * This is the second or subsequent kprobe at the address - handle
 577  * the intricacies
 578  */
 579 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
 580                                           struct kprobe *p)
 581 {
 582         int ret = 0;
 583         struct kprobe *ap = old_p;
 584
 585         if (old_p->pre_handler != aggr_pre_handler) {
 586                 /* If old_p is not an aggr_probe, create new aggr_kprobe. */
 587                 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 588                 if (!ap)
 589                         return -ENOMEM;
 590                 add_aggr_kprobe(ap, old_p);
 591         }
 592
 593         if (kprobe_gone(ap)) {
 594                 /*
 595                  * Attempting to insert new probe at the same location that
 596                  * had a probe in the module vaddr area which already
 597                  * freed. So, the instruction slot has already been
 598                  * released. We need a new slot for the new probe.
 599                  */
 600                 ret = arch_prepare_kprobe(ap);
 601                 if (ret)
 602                         /*
 603                          * Even if fail to allocate new slot, don't need to
 604                          * free aggr_probe. It will be used next time, or
 605                          * freed by unregister_kprobe.
 606                          */
 607                         return ret;
 608
 609                 /*
 610                  * Clear gone flag to prevent allocating new slot again, and
 611                  * set disabled flag because it is not armed yet.
 612                  */
 613                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
 614                             | KPROBE_FLAG_DISABLED;
 615         }
 616
 617         copy_kprobe(ap, p);
 618         return add_new_kprobe(ap, p);
 619 }
 620
 621 /* Try to disable aggr_kprobe, and return 1 if succeeded.*/
 622 static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
 623 {
 624         struct kprobe *kp;
 625
 626         list_for_each_entry_rcu(kp, &p->list, list) {
 627                 if (!kprobe_disabled(kp))
 628                         /*
 629                          * There is an active probe on the list.
 630                          * We can't disable aggr_kprobe.
 631                          */
 632                         return 0;
 633         }
 634         p->flags |= KPROBE_FLAG_DISABLED;
 635         return 1;
 636 }
 637
 638 static int __kprobes in_kprobes_functions(unsigned long addr)
 639 {
 640         struct kprobe_blackpoint *kb;
 641
 642         if (addr >= (unsigned long)__kprobes_text_start &&
 643             addr < (unsigned long)__kprobes_text_end)
 644                 return -EINVAL;
 645         /*
 646          * If there exists a kprobe_blacklist, verify and
 647          * fail any probe registration in the prohibited area
 648          */
 649         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
 650                 if (kb->start_addr) {
 651                         if (addr >= kb->start_addr &&
 652                             addr < (kb->start_addr + kb->range))
 653                                 return -EINVAL;
 654                 }
 655         }
 656         return 0;
 657 }
 658
 659 /*
 660  * If we have a symbol_name argument, look it up and add the offset field
 661  * to it. This way, we can specify a relative address to a symbol.
 662  */
 663 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
 664 {
 665         kprobe_opcode_t *addr = p->addr;
 666         if (p->symbol_name) {
 667                 if (addr)
 668                         return NULL;
 669                 kprobe_lookup_name(p->symbol_name, addr);
 670         }
 671
 672         if (!addr)
 673                 return NULL;
 674         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
 675 }
 676
 677 int __kprobes register_kprobe(struct kprobe *p)
 678 {
 679         int ret = 0;
 680         struct kprobe *old_p;
 681         struct module *probed_mod;
 682         kprobe_opcode_t *addr;
 683
 684         addr = kprobe_addr(p);
 685         if (!addr)
 686                 return -EINVAL;
 687         p->addr = addr;
 688
 689         preempt_disable();
 690         if (!kernel_text_address((unsigned long) p->addr) ||
 691             in_kprobes_functions((unsigned long) p->addr)) {
 692                 preempt_enable();
 693                 return -EINVAL;
 694         }
 695
 696         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
 697         p->flags &= KPROBE_FLAG_DISABLED;
 698
 699         /*
 700          * Check if are we probing a module.
 701          */
 702         probed_mod = __module_text_address((unsigned long) p->addr);
 703         if (probed_mod) {
 704                 /*
 705                  * We must hold a refcount of the probed module while updating
 706                  * its code to prohibit unexpected unloading.
 707                  */
 708                 if (unlikely(!try_module_get(probed_mod))) {
 709                         preempt_enable();
 710                         return -EINVAL;
 711                 }
 712                 /*
 713                  * If the module freed .init.text, we couldn't insert
 714                  * kprobes in there.
 715                  */
 716                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
 717                     probed_mod->state != MODULE_STATE_COMING) {
 718                         module_put(probed_mod);
 719                         preempt_enable();
 720                         return -EINVAL;
 721                 }
 722         }
 723         preempt_enable();
 724
 725         p->nmissed = 0;
 726         INIT_LIST_HEAD(&p->list);
 727         mutex_lock(&kprobe_mutex);
 728         old_p = get_kprobe(p->addr);
 729         if (old_p) {
 730                 ret = register_aggr_kprobe(old_p, p);
 731                 goto out;
 732         }
 733
 734         mutex_lock(&text_mutex);
 735         ret = arch_prepare_kprobe(p);
 736         if (ret)
 737                 goto out_unlock_text;
 738
 739         INIT_HLIST_NODE(&p->hlist);
 740         hlist_add_head_rcu(&p->hlist,
 741                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
 742
 743         if (!kprobes_all_disarmed && !kprobe_disabled(p))
 744                 arch_arm_kprobe(p);
 745
 746 out_unlock_text:
 747         mutex_unlock(&text_mutex);
 748 out:
 749         mutex_unlock(&kprobe_mutex);
 750
 751         if (probed_mod)
 752                 module_put(probed_mod);
 753
 754         return ret;
 755 }
 756 EXPORT_SYMBOL_GPL(register_kprobe);
 757
 758 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
 759 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
 760 {
 761         struct kprobe *old_p, *list_p;
 762
 763         old_p = get_kprobe(p->addr);
 764         if (unlikely(!old_p))
 765                 return NULL;
 766
 767         if (p != old_p) {
 768                 list_for_each_entry_rcu(list_p, &old_p->list, list)
 769                         if (list_p == p)
 770                         /* kprobe p is a valid probe */
 771                                 goto valid;
 772                 return NULL;
 773         }
 774 valid:
 775         return old_p;
 776 }
 777
 778 /*
 779  * Unregister a kprobe without a scheduler synchronization.
 780  */
 781 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
 782 {
 783         struct kprobe *old_p, *list_p;
 784
 785         old_p = __get_valid_kprobe(p);
 786         if (old_p == NULL)
 787                 return -EINVAL;
 788
 789         if (old_p == p ||
 790             (old_p->pre_handler == aggr_pre_handler &&
 791              list_is_singular(&old_p->list))) {
 792                 /*
 793                  * Only probe on the hash list. Disarm only if kprobes are
 794                  * enabled and not gone - otherwise, the breakpoint would
 795                  * already have been removed. We save on flushing icache.
 796                  */
 797                 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
 798                         disarm_kprobe(p);
 799                 hlist_del_rcu(&old_p->hlist);
 800         } else {
 801                 if (p->break_handler && !kprobe_gone(p))
 802                         old_p->break_handler = NULL;
 803                 if (p->post_handler && !kprobe_gone(p)) {
 804                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
 805                                 if ((list_p != p) && (list_p->post_handler))
 806                                         goto noclean;
 807                         }
 808                         old_p->post_handler = NULL;
 809                 }
 810 noclean:
 811                 list_del_rcu(&p->list);
 812                 if (!kprobe_disabled(old_p)) {
 813                         try_to_disable_aggr_kprobe(old_p);
 814                         if (!kprobes_all_disarmed && kprobe_disabled(old_p))
 815                                 disarm_kprobe(old_p);
 816                 }
 817         }
 818         return 0;
 819 }
 820
 821 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
 822 {
 823         struct kprobe *old_p;
 824
 825         if (list_empty(&p->list))
 826                 arch_remove_kprobe(p);
 827         else if (list_is_singular(&p->list)) {
 828                 /* "p" is the last child of an aggr_kprobe */
 829                 old_p = list_entry(p->list.next, struct kprobe, list);
 830                 list_del(&p->list);
 831                 arch_remove_kprobe(old_p);
 832                 kfree(old_p);
 833         }
 834 }
 835
 836 int __kprobes register_kprobes(struct kprobe **kps, int num)
 837 {
 838         int i, ret = 0;
 839
 840         if (num <= 0)
 841                 return -EINVAL;
 842         for (i = 0; i < num; i++) {
 843                 ret = register_kprobe(kps[i]);
 844                 if (ret < 0) {
 845                         if (i > 0)
 846                                 unregister_kprobes(kps, i);
 847                         break;
 848                 }
 849         }
 850         return ret;
 851 }
 852 EXPORT_SYMBOL_GPL(register_kprobes);
 853
 854 void __kprobes unregister_kprobe(struct kprobe *p)
 855 {
 856         unregister_kprobes(&p, 1);
 857 }
 858 EXPORT_SYMBOL_GPL(unregister_kprobe);
 859
 860 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
 861 {
 862         int i;
 863
 864         if (num <= 0)
 865                 return;
 866         mutex_lock(&kprobe_mutex);
 867         for (i = 0; i < num; i++)
 868                 if (__unregister_kprobe_top(kps[i]) < 0)
 869                         kps[i]->addr = NULL;
 870         mutex_unlock(&kprobe_mutex);
 871
 872         synchronize_sched();
 873         for (i = 0; i < num; i++)
 874                 if (kps[i]->addr)
 875                         __unregister_kprobe_bottom(kps[i]);
 876 }
 877 EXPORT_SYMBOL_GPL(unregister_kprobes);
 878
 879 static struct notifier_block kprobe_exceptions_nb = {
 880         .notifier_call = kprobe_exceptions_notify,
 881         .priority = 0x7fffffff /* we need to be notified first */
 882 };
 883
 884 unsigned long __weak arch_deref_entry_point(void *entry)
 885 {
 886         return (unsigned long)entry;
 887 }
 888
 889 int __kprobes register_jprobes(struct jprobe **jps, int num)
 890 {
 891         struct jprobe *jp;
 892         int ret = 0, i;
 893
 894         if (num <= 0)
 895                 return -EINVAL;
 896         for (i = 0; i < num; i++) {
 897                 unsigned long addr;
 898                 jp = jps[i];
 899                 addr = arch_deref_entry_point(jp->entry);
 900
 901                 if (!kernel_text_address(addr))
 902                         ret = -EINVAL;
 903                 else {
 904                         /* Todo: Verify probepoint is a function entry point */
 905                         jp->kp.pre_handler = setjmp_pre_handler;
 906                         jp->kp.break_handler = longjmp_break_handler;
 907                         ret = register_kprobe(&jp->kp);
 908                 }
 909                 if (ret < 0) {
 910                         if (i > 0)
 911                                 unregister_jprobes(jps, i);
 912                         break;
 913                 }
 914         }
 915         return ret;
 916 }
 917 EXPORT_SYMBOL_GPL(register_jprobes);
 918
 919 int __kprobes register_jprobe(struct jprobe *jp)
 920 {
 921         return register_jprobes(&jp, 1);
 922 }
 923 EXPORT_SYMBOL_GPL(register_jprobe);
 924
 925 void __kprobes unregister_jprobe(struct jprobe *jp)
 926 {
 927         unregister_jprobes(&jp, 1);
 928 }
 929 EXPORT_SYMBOL_GPL(unregister_jprobe);
 930
 931 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
 932 {
 933         int i;
 934
 935         if (num <= 0)
 936                 return;
 937         mutex_lock(&kprobe_mutex);
 938         for (i = 0; i < num; i++)
 939                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
 940                         jps[i]->kp.addr = NULL;
 941         mutex_unlock(&kprobe_mutex);
 942
 943         synchronize_sched();
 944         for (i = 0; i < num; i++) {
 945                 if (jps[i]->kp.addr)
 946                         __unregister_kprobe_bottom(&jps[i]->kp);
 947         }
 948 }
 949 EXPORT_SYMBOL_GPL(unregister_jprobes);
 950
 951 #ifdef CONFIG_KRETPROBES
 952 /*
 953  * This kprobe pre_handler is registered with every kretprobe. When probe
 954  * hits it will set up the return probe.
 955  */
 956 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
 957                                            struct pt_regs *regs)
 958 {
 959         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
 960         unsigned long hash, flags = 0;
 961         struct kretprobe_instance *ri;
 962
 963         /*TODO: consider to only swap the RA after the last pre_handler fired */
 964         hash = hash_ptr(current, KPROBE_HASH_BITS);
 965         spin_lock_irqsave(&rp->lock, flags);
 966         if (!hlist_empty(&rp->free_instances)) {
 967                 ri = hlist_entry(rp->free_instances.first,
 968                                 struct kretprobe_instance, hlist);
 969                 hlist_del(&ri->hlist);
 970                 spin_unlock_irqrestore(&rp->lock, flags);
 971
 972                 ri->rp = rp;
 973                 ri->task = current;
 974
 975                 if (rp->entry_handler && rp->entry_handler(ri, regs))
 976                         return 0;
 977
 978                 arch_prepare_kretprobe(ri, regs);
 979
 980                 /* XXX(hch): why is there no hlist_move_head? */
 981                 INIT_HLIST_NODE(&ri->hlist);
 982                 kretprobe_table_lock(hash, &flags);
 983                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
 984                 kretprobe_table_unlock(hash, &flags);
 985         } else {
 986                 rp->nmissed++;
 987                 spin_unlock_irqrestore(&rp->lock, flags);
 988         }
 989         return 0;
 990 }
 991
 992 int __kprobes register_kretprobe(struct kretprobe *rp)
 993 {
 994         int ret = 0;
 995         struct kretprobe_instance *inst;
 996         int i;
 997         void *addr;
 998
 999         if (kretprobe_blacklist_size) {
1000                 addr = kprobe_addr(&rp->kp);
1001                 if (!addr)
1002                         return -EINVAL;
1003
1004                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1005                         if (kretprobe_blacklist[i].addr == addr)
1006                                 return -EINVAL;
1007                 }
1008         }
1009
1010         rp->kp.pre_handler = pre_handler_kretprobe;
1011         rp->kp.post_handler = NULL;
1012         rp->kp.fault_handler = NULL;
1013         rp->kp.break_handler = NULL;
1014
1015         /* Pre-allocate memory for max kretprobe instances */
1016         if (rp->maxactive <= 0) {
1017 #ifdef CONFIG_PREEMPT
1018                 rp->maxactive = max(10, 2 * NR_CPUS);
1019 #else
1020                 rp->maxactive = NR_CPUS;
1021 #endif
1022         }
1023         spin_lock_init(&rp->lock);
1024         INIT_HLIST_HEAD(&rp->free_instances);
1025         for (i = 0; i < rp->maxactive; i++) {
1026                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1027                                rp->data_size, GFP_KERNEL);
1028                 if (inst == NULL) {
1029                         free_rp_inst(rp);
1030                         return -ENOMEM;
1031                 }
1032                 INIT_HLIST_NODE(&inst->hlist);
1033                 hlist_add_head(&inst->hlist, &rp->free_instances);
1034         }
1035
1036         rp->nmissed = 0;
1037         /* Establish function entry probe point */
1038         ret = register_kprobe(&rp->kp);
1039         if (ret != 0)
1040                 free_rp_inst(rp);
1041         return ret;
1042 }
1043 EXPORT_SYMBOL_GPL(register_kretprobe);
1044
1045 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1046 {
1047         int ret = 0, i;
1048
1049         if (num <= 0)
1050                 return -EINVAL;
1051         for (i = 0; i < num; i++) {
1052                 ret = register_kretprobe(rps[i]);
1053                 if (ret < 0) {
1054                         if (i > 0)
1055                                 unregister_kretprobes(rps, i);
1056                         break;
1057                 }
1058         }
1059         return ret;
1060 }
1061 EXPORT_SYMBOL_GPL(register_kretprobes);
1062
1063 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1064 {
1065         unregister_kretprobes(&rp, 1);
1066 }
1067 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1068
1069 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1070 {
1071         int i;
1072
1073         if (num <= 0)
1074                 return;
1075         mutex_lock(&kprobe_mutex);
1076         for (i = 0; i < num; i++)
1077                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1078                         rps[i]->kp.addr = NULL;
1079         mutex_unlock(&kprobe_mutex);
1080
1081         synchronize_sched();
1082         for (i = 0; i < num; i++) {
1083                 if (rps[i]->kp.addr) {
1084                         __unregister_kprobe_bottom(&rps[i]->kp);
1085                         cleanup_rp_inst(rps[i]);
1086                 }
1087         }
1088 }
1089 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1090
1091 #else /* CONFIG_KRETPROBES */
1092 int __kprobes register_kretprobe(struct kretprobe *rp)
1093 {
1094         return -ENOSYS;
1095 }
1096 EXPORT_SYMBOL_GPL(register_kretprobe);
1097
1098 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1099 {
1100         return -ENOSYS;
1101 }
1102 EXPORT_SYMBOL_GPL(register_kretprobes);
1103
1104 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1105 {
1106 }
1107 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1108
1109 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1110 {
1111 }
1112 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1113
1114 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1115                                            struct pt_regs *regs)
1116 {
1117         return 0;
1118 }
1119
1120 #endif /* CONFIG_KRETPROBES */
1121
1122 /* Set the kprobe gone and remove its instruction buffer. */
1123 static void __kprobes kill_kprobe(struct kprobe *p)
1124 {
1125         struct kprobe *kp;
1126
1127         p->flags |= KPROBE_FLAG_GONE;
1128         if (p->pre_handler == aggr_pre_handler) {
1129                 /*
1130                  * If this is an aggr_kprobe, we have to list all the
1131                  * chained probes and mark them GONE.
1132                  */
1133                 list_for_each_entry_rcu(kp, &p->list, list)
1134                         kp->flags |= KPROBE_FLAG_GONE;
1135                 p->post_handler = NULL;
1136                 p->break_handler = NULL;
1137         }
1138         /*
1139          * Here, we can remove insn_slot safely, because no thread calls
1140          * the original probed function (which will be freed soon) any more.
1141          */
1142         arch_remove_kprobe(p);
1143 }
1144
1145 void __kprobes dump_kprobe(struct kprobe *kp)
1146 {
1147         printk(KERN_WARNING "Dumping kprobe:\n");
1148         printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1149                kp->symbol_name, kp->addr, kp->offset);
1150 }
1151
1152 /* Module notifier call back, checking kprobes on the module */
1153 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1154                                              unsigned long val, void *data)
1155 {
1156         struct module *mod = data;
1157         struct hlist_head *head;
1158         struct hlist_node *node;
1159         struct kprobe *p;
1160         unsigned int i;
1161         int checkcore = (val == MODULE_STATE_GOING);
1162
1163         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1164                 return NOTIFY_DONE;
1165
1166         /*
1167          * When MODULE_STATE_GOING was notified, both of module .text and
1168          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1169          * notified, only .init.text section would be freed. We need to
1170          * disable kprobes which have been inserted in the sections.
1171          */
1172         mutex_lock(&kprobe_mutex);
1173         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1174                 head = &kprobe_table[i];
1175                 hlist_for_each_entry_rcu(p, node, head, hlist)
1176                         if (within_module_init((unsigned long)p->addr, mod) ||
1177                             (checkcore &&
1178                              within_module_core((unsigned long)p->addr, mod))) {
1179                                 /*
1180                                  * The vaddr this probe is installed will soon
1181                                  * be vfreed buy not synced to disk. Hence,
1182                                  * disarming the breakpoint isn't needed.
1183                                  */
1184                                 kill_kprobe(p);
1185                         }
1186         }
1187         mutex_unlock(&kprobe_mutex);
1188         return NOTIFY_DONE;
1189 }
1190
1191 static struct notifier_block kprobe_module_nb = {
1192         .notifier_call = kprobes_module_callback,
1193         .priority = 0
1194 };
1195
1196 static int __init init_kprobes(void)
1197 {
1198         int i, err = 0;
1199         unsigned long offset = 0, size = 0;
1200         char *modname, namebuf[128];
1201         const char *symbol_name;
1202         void *addr;
1203         struct kprobe_blackpoint *kb;
1204
1205         /* FIXME allocate the probe table, currently defined statically */
1206         /* initialize all list heads */
1207         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1208                 INIT_HLIST_HEAD(&kprobe_table[i]);
1209                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1210                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1211         }
1212
1213         /*
1214          * Lookup and populate the kprobe_blacklist.
1215          *
1216          * Unlike the kretprobe blacklist, we'll need to determine
1217          * the range of addresses that belong to the said functions,
1218          * since a kprobe need not necessarily be at the beginning
1219          * of a function.
1220          */
1221         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1222                 kprobe_lookup_name(kb->name, addr);
1223                 if (!addr)
1224                         continue;
1225
1226                 kb->start_addr = (unsigned long)addr;
1227                 symbol_name = kallsyms_lookup(kb->start_addr,
1228                                 &size, &offset, &modname, namebuf);
1229                 if (!symbol_name)
1230                         kb->range = 0;
1231                 else
1232                         kb->range = size;
1233         }
1234
1235         if (kretprobe_blacklist_size) {
1236                 /* lookup the function address from its name */
1237                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1238                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1239                                            kretprobe_blacklist[i].addr);
1240                         if (!kretprobe_blacklist[i].addr)
1241                                 printk("kretprobe: lookup failed: %s\n",
1242                                        kretprobe_blacklist[i].name);
1243                 }
1244         }
1245
1246         /* By default, kprobes are armed */
1247         kprobes_all_disarmed = false;
1248
1249         err = arch_init_kprobes();
1250         if (!err)
1251                 err = register_die_notifier(&kprobe_exceptions_nb);
1252         if (!err)
1253                 err = register_module_notifier(&kprobe_module_nb);
1254
1255         kprobes_initialized = (err == 0);
1256
1257         if (!err)
1258                 init_test_probes();
1259         return err;
1260 }
1261
1262 #ifdef CONFIG_DEBUG_FS
1263 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1264                 const char *sym, int offset,char *modname)
1265 {
1266         char *kprobe_type;
1267
1268         if (p->pre_handler == pre_handler_kretprobe)
1269                 kprobe_type = "r";
1270         else if (p->pre_handler == setjmp_pre_handler)
1271                 kprobe_type = "j";
1272         else
1273                 kprobe_type = "k";
1274         if (sym)
1275                 seq_printf(pi, "%p  %s  %s+0x%x  %s %s%s\n",
1276                         p->addr, kprobe_type, sym, offset,
1277                         (modname ? modname : " "),
1278                         (kprobe_gone(p) ? "[GONE]" : ""),
1279                         ((kprobe_disabled(p) && !kprobe_gone(p)) ?
1280                          "[DISABLED]" : ""));
1281         else
1282                 seq_printf(pi, "%p  %s  %p %s%s\n",
1283                         p->addr, kprobe_type, p->addr,
1284                         (kprobe_gone(p) ? "[GONE]" : ""),
1285                         ((kprobe_disabled(p) && !kprobe_gone(p)) ?
1286                          "[DISABLED]" : ""));
1287 }
1288
1289 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1290 {
1291         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1292 }
1293
1294 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1295 {
1296         (*pos)++;
1297         if (*pos >= KPROBE_TABLE_SIZE)
1298                 return NULL;
1299         return pos;
1300 }
1301
1302 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1303 {
1304         /* Nothing to do */
1305 }
1306
1307 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1308 {
1309         struct hlist_head *head;
1310         struct hlist_node *node;
1311         struct kprobe *p, *kp;
1312         const char *sym = NULL;
1313         unsigned int i = *(loff_t *) v;
1314         unsigned long offset = 0;
1315         char *modname, namebuf[128];
1316
1317         head = &kprobe_table[i];
1318         preempt_disable();
1319         hlist_for_each_entry_rcu(p, node, head, hlist) {
1320                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1321                                         &offset, &modname, namebuf);
1322                 if (p->pre_handler == aggr_pre_handler) {
1323                         list_for_each_entry_rcu(kp, &p->list, list)
1324                                 report_probe(pi, kp, sym, offset, modname);
1325                 } else
1326                         report_probe(pi, p, sym, offset, modname);
1327         }
1328         preempt_enable();
1329         return 0;
1330 }
1331
1332 static struct seq_operations kprobes_seq_ops = {
1333         .start = kprobe_seq_start,
1334         .next  = kprobe_seq_next,
1335         .stop  = kprobe_seq_stop,
1336         .show  = show_kprobe_addr
1337 };
1338
1339 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1340 {
1341         return seq_open(filp, &kprobes_seq_ops);
1342 }
1343
1344 static struct file_operations debugfs_kprobes_operations = {
1345         .open           = kprobes_open,
1346         .read           = seq_read,
1347         .llseek         = seq_lseek,
1348         .release        = seq_release,
1349 };
1350
1351 /* Disable one kprobe */
1352 int __kprobes disable_kprobe(struct kprobe *kp)
1353 {
1354         int ret = 0;
1355         struct kprobe *p;
1356
1357         mutex_lock(&kprobe_mutex);
1358
1359         /* Check whether specified probe is valid. */
1360         p = __get_valid_kprobe(kp);
1361         if (unlikely(p == NULL)) {
1362                 ret = -EINVAL;
1363                 goto out;
1364         }
1365
1366         /* If the probe is already disabled (or gone), just return */
1367         if (kprobe_disabled(kp))
1368                 goto out;
1369
1370         kp->flags |= KPROBE_FLAG_DISABLED;
1371         if (p != kp)
1372                 /* When kp != p, p is always enabled. */
1373                 try_to_disable_aggr_kprobe(p);
1374
1375         if (!kprobes_all_disarmed && kprobe_disabled(p))
1376                 disarm_kprobe(p);
1377 out:
1378         mutex_unlock(&kprobe_mutex);
1379         return ret;
1380 }
1381 EXPORT_SYMBOL_GPL(disable_kprobe);
1382
1383 /* Enable one kprobe */
1384 int __kprobes enable_kprobe(struct kprobe *kp)
1385 {
1386         int ret = 0;
1387         struct kprobe *p;
1388
1389         mutex_lock(&kprobe_mutex);
1390
1391         /* Check whether specified probe is valid. */
1392         p = __get_valid_kprobe(kp);
1393         if (unlikely(p == NULL)) {
1394                 ret = -EINVAL;
1395                 goto out;
1396         }
1397
1398         if (kprobe_gone(kp)) {
1399                 /* This kprobe has gone, we couldn't enable it. */
1400                 ret = -EINVAL;
1401                 goto out;
1402         }
1403
1404         if (!kprobes_all_disarmed && kprobe_disabled(p))
1405                 arm_kprobe(p);
1406
1407         p->flags &= ~KPROBE_FLAG_DISABLED;
1408         if (p != kp)
1409                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1410 out:
1411         mutex_unlock(&kprobe_mutex);
1412         return ret;
1413 }
1414 EXPORT_SYMBOL_GPL(enable_kprobe);
1415
1416 static void __kprobes arm_all_kprobes(void)
1417 {
1418         struct hlist_head *head;
1419         struct hlist_node *node;
1420         struct kprobe *p;
1421         unsigned int i;
1422
1423         mutex_lock(&kprobe_mutex);
1424
1425         /* If kprobes are armed, just return */
1426         if (!kprobes_all_disarmed)
1427                 goto already_enabled;
1428
1429         mutex_lock(&text_mutex);
1430         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1431                 head = &kprobe_table[i];
1432                 hlist_for_each_entry_rcu(p, node, head, hlist)
1433                         if (!kprobe_disabled(p))
1434                                 arch_arm_kprobe(p);
1435         }
1436         mutex_unlock(&text_mutex);
1437
1438         kprobes_all_disarmed = false;
1439         printk(KERN_INFO "Kprobes globally enabled\n");
1440
1441 already_enabled:
1442         mutex_unlock(&kprobe_mutex);
1443         return;
1444 }
1445
1446 static void __kprobes disarm_all_kprobes(void)
1447 {
1448         struct hlist_head *head;
1449         struct hlist_node *node;
1450         struct kprobe *p;
1451         unsigned int i;
1452
1453         mutex_lock(&kprobe_mutex);
1454
1455         /* If kprobes are already disarmed, just return */
1456         if (kprobes_all_disarmed)
1457                 goto already_disabled;
1458
1459         kprobes_all_disarmed = true;
1460         printk(KERN_INFO "Kprobes globally disabled\n");
1461         mutex_lock(&text_mutex);
1462         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1463                 head = &kprobe_table[i];
1464                 hlist_for_each_entry_rcu(p, node, head, hlist) {
1465                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1466                                 arch_disarm_kprobe(p);
1467                 }
1468         }
1469
1470         mutex_unlock(&text_mutex);
1471         mutex_unlock(&kprobe_mutex);
1472         /* Allow all currently running kprobes to complete */
1473         synchronize_sched();
1474         return;
1475
1476 already_disabled:
1477         mutex_unlock(&kprobe_mutex);
1478         return;
1479 }
1480
1481 /*
1482  * XXX: The debugfs bool file interface doesn't allow for callbacks
1483  * when the bool state is switched. We can reuse that facility when
1484  * available
1485  */
1486 static ssize_t read_enabled_file_bool(struct file *file,
1487                char __user *user_buf, size_t count, loff_t *ppos)
1488 {
1489         char buf[3];
1490
1491         if (!kprobes_all_disarmed)
1492                 buf[0] = '1';
1493         else
1494                 buf[0] = '0';
1495         buf[1] = '\n';
1496         buf[2] = 0x00;
1497         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1498 }
1499
1500 static ssize_t write_enabled_file_bool(struct file *file,
1501                const char __user *user_buf, size_t count, loff_t *ppos)
1502 {
1503         char buf[32];
1504         int buf_size;
1505
1506         buf_size = min(count, (sizeof(buf)-1));
1507         if (copy_from_user(buf, user_buf, buf_size))
1508                 return -EFAULT;
1509
1510         switch (buf[0]) {
1511         case 'y':
1512         case 'Y':
1513         case '1':
1514                 arm_all_kprobes();
1515                 break;
1516         case 'n':
1517         case 'N':
1518         case '0':
1519                 disarm_all_kprobes();
1520                 break;
1521         }
1522
1523         return count;
1524 }
1525
1526 static struct file_operations fops_kp = {
1527         .read =         read_enabled_file_bool,
1528         .write =        write_enabled_file_bool,
1529 };
1530
1531 static int __kprobes debugfs_kprobe_init(void)
1532 {
1533         struct dentry *dir, *file;
1534         unsigned int value = 1;
1535
1536         dir = debugfs_create_dir("kprobes", NULL);
1537         if (!dir)
1538                 return -ENOMEM;
1539
1540         file = debugfs_create_file("list", 0444, dir, NULL,
1541                                 &debugfs_kprobes_operations);
1542         if (!file) {
1543                 debugfs_remove(dir);
1544                 return -ENOMEM;
1545         }
1546
1547         file = debugfs_create_file("enabled", 0600, dir,
1548                                         &value, &fops_kp);
1549         if (!file) {
1550                 debugfs_remove(dir);
1551                 return -ENOMEM;
1552         }
1553
1554         return 0;
1555 }
1556
1557 late_initcall(debugfs_kprobe_init);
1558 #endif /* CONFIG_DEBUG_FS */
1559
1560 module_init(init_kprobes);
1561
1562 /* defined in arch/.../kernel/kprobes.c */
1563 EXPORT_SYMBOL_GPL(jprobe_return);