arch/ia64/kernel/unwind.c

   1 /*
   2  * Copyright (C) 1999-2003 Hewlett-Packard Co
   3  *      David Mosberger-Tang <davidm@hpl.hp.com>
   4  */
   5 /*
   6  * This file implements call frame unwind support for the Linux
   7  * kernel.  Parsing and processing the unwind information is
   8  * time-consuming, so this implementation translates the unwind
   9  * descriptors into unwind scripts.  These scripts are very simple
  10  * (basically a sequence of assignments) and efficient to execute.
  11  * They are cached for later re-use.  Each script is specific for a
  12  * given instruction pointer address and the set of predicate values
  13  * that the script depends on (most unwind descriptors are
  14  * unconditional and scripts often do not depend on predicates at
  15  * all).  This code is based on the unwind conventions described in
  16  * the "IA-64 Software Conventions and Runtime Architecture" manual.
  17  *
  18  * SMP conventions:
  19  *      o updates to the global unwind data (in structure "unw") are serialized
  20  *        by the unw.lock spinlock
  21  *      o each unwind script has its own read-write lock; a thread must acquire
  22  *        a read lock before executing a script and must acquire a write lock
  23  *        before modifying a script
  24  *      o if both the unw.lock spinlock and a script's read-write lock must be
  25  *        acquired, then the read-write lock must be acquired first.
  26  */
  27 #include <linux/bootmem.h>
  28 #include <linux/kernel.h>
  29 #include <linux/sched.h>
  30 #include <linux/slab.h>
  31
  32 #include <asm/unwind.h>
  33
  34 #include <asm/delay.h>
  35 #include <asm/page.h>
  36 #include <asm/ptrace.h>
  37 #include <asm/ptrace_offsets.h>
  38 #include <asm/rse.h>
  39 #include <asm/system.h>
  40 #include <asm/uaccess.h>
  41
  42 #include "entry.h"
  43 #include "unwind_i.h"
  44
  45 #define MIN(a,b)        ((a) < (b) ? (a) : (b))
  46 #define p5              5
  47
  48 #define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  49 #define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  50
  51 #define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  52 #define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  53
  54 #define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  55
  56 #ifdef UNW_DEBUG
  57   static unsigned int unw_debug_level = UNW_DEBUG;
  58 #  ifdef CONFIG_KDB
  59 #    include <linux/kdb.h>
  60 #    define UNW_DEBUG_ON(n)     (unw_debug_level >= n && !KDB_IS_RUNNING())
  61 #    define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) kdb_printf(__VA_ARGS__)
  62 #  else /* !CONFIG_KDB */
  63 #    define UNW_DEBUG_ON(n)     unw_debug_level >= n
  64      /* Do not code a printk level, not all debug lines end in newline */
  65 #    define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  66 #  endif /* CONFIG_KDB */
  67 #  define inline
  68 #else /* !UNW_DEBUG */
  69 #  define UNW_DEBUG_ON(n)  0
  70 #  define UNW_DPRINT(n, ...)
  71 #endif /* UNW_DEBUG */
  72
  73 #if UNW_STATS
  74 # define STAT(x...)     x
  75 #else
  76 # define STAT(x...)
  77 #endif
  78
  79 #define alloc_reg_state()       kmalloc(sizeof(struct unw_state_record), GFP_ATOMIC)
  80 #define free_reg_state(usr)     kfree(usr)
  81 #define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  82 #define free_labeled_state(usr) kfree(usr)
  83
  84 typedef unsigned long unw_word;
  85 typedef unsigned char unw_hash_index_t;
  86
  87 #define struct_offset(str,fld)  ((char *)&((str *)NULL)->fld - (char *) 0)
  88
  89 static struct {
  90         spinlock_t lock;                        /* spinlock for unwind data */
  91
  92         /* list of unwind tables (one per load-module) */
  93         struct unw_table *tables;
  94
  95         /* table of registers that prologues can save (and order in which they're saved): */
  96         const unsigned char save_order[8];
  97
  98         /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
  99         unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
 100
 101         unsigned short lru_head;                /* index of lead-recently used script */
 102         unsigned short lru_tail;                /* index of most-recently used script */
 103
 104         /* index into unw_frame_info for preserved register i */
 105         unsigned short preg_index[UNW_NUM_REGS];
 106
 107         /* unwind table for the kernel: */
 108         struct unw_table kernel_table;
 109
 110         /* unwind table describing the gate page (kernel code that is mapped into user space): */
 111         size_t gate_table_size;
 112         unsigned long *gate_table;
 113
 114         /* hash table that maps instruction pointer to script index: */
 115         unsigned short hash[UNW_HASH_SIZE];
 116
 117         /* script cache: */
 118         struct unw_script cache[UNW_CACHE_SIZE];
 119
 120 # ifdef UNW_DEBUG
 121         const char *preg_name[UNW_NUM_REGS];
 122 # endif
 123 # if UNW_STATS
 124         struct {
 125                 struct {
 126                         int lookups;
 127                         int hinted_hits;
 128                         int normal_hits;
 129                         int collision_chain_traversals;
 130                 } cache;
 131                 struct {
 132                         unsigned long build_time;
 133                         unsigned long run_time;
 134                         unsigned long parse_time;
 135                         int builds;
 136                         int news;
 137                         int collisions;
 138                         int runs;
 139                 } script;
 140                 struct {
 141                         unsigned long init_time;
 142                         unsigned long unwind_time;
 143                         int inits;
 144                         int unwinds;
 145                 } api;
 146         } stat;
 147 # endif
 148 } unw = {
 149         .tables = &unw.kernel_table,
 150         .lock = SPIN_LOCK_UNLOCKED,
 151         .save_order = {
 152                 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 153                 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 154         },
 155         .preg_index = {
 156                 struct_offset(struct unw_frame_info, pri_unat_loc)/8,   /* PRI_UNAT_GR */
 157                 struct_offset(struct unw_frame_info, pri_unat_loc)/8,   /* PRI_UNAT_MEM */
 158                 struct_offset(struct unw_frame_info, bsp_loc)/8,
 159                 struct_offset(struct unw_frame_info, bspstore_loc)/8,
 160                 struct_offset(struct unw_frame_info, pfs_loc)/8,
 161                 struct_offset(struct unw_frame_info, rnat_loc)/8,
 162                 struct_offset(struct unw_frame_info, psp)/8,
 163                 struct_offset(struct unw_frame_info, rp_loc)/8,
 164                 struct_offset(struct unw_frame_info, r4)/8,
 165                 struct_offset(struct unw_frame_info, r5)/8,
 166                 struct_offset(struct unw_frame_info, r6)/8,
 167                 struct_offset(struct unw_frame_info, r7)/8,
 168                 struct_offset(struct unw_frame_info, unat_loc)/8,
 169                 struct_offset(struct unw_frame_info, pr_loc)/8,
 170                 struct_offset(struct unw_frame_info, lc_loc)/8,
 171                 struct_offset(struct unw_frame_info, fpsr_loc)/8,
 172                 struct_offset(struct unw_frame_info, b1_loc)/8,
 173                 struct_offset(struct unw_frame_info, b2_loc)/8,
 174                 struct_offset(struct unw_frame_info, b3_loc)/8,
 175                 struct_offset(struct unw_frame_info, b4_loc)/8,
 176                 struct_offset(struct unw_frame_info, b5_loc)/8,
 177                 struct_offset(struct unw_frame_info, f2_loc)/8,
 178                 struct_offset(struct unw_frame_info, f3_loc)/8,
 179                 struct_offset(struct unw_frame_info, f4_loc)/8,
 180                 struct_offset(struct unw_frame_info, f5_loc)/8,
 181                 struct_offset(struct unw_frame_info, fr_loc[16 - 16])/8,
 182                 struct_offset(struct unw_frame_info, fr_loc[17 - 16])/8,
 183                 struct_offset(struct unw_frame_info, fr_loc[18 - 16])/8,
 184                 struct_offset(struct unw_frame_info, fr_loc[19 - 16])/8,
 185                 struct_offset(struct unw_frame_info, fr_loc[20 - 16])/8,
 186                 struct_offset(struct unw_frame_info, fr_loc[21 - 16])/8,
 187                 struct_offset(struct unw_frame_info, fr_loc[22 - 16])/8,
 188                 struct_offset(struct unw_frame_info, fr_loc[23 - 16])/8,
 189                 struct_offset(struct unw_frame_info, fr_loc[24 - 16])/8,
 190                 struct_offset(struct unw_frame_info, fr_loc[25 - 16])/8,
 191                 struct_offset(struct unw_frame_info, fr_loc[26 - 16])/8,
 192                 struct_offset(struct unw_frame_info, fr_loc[27 - 16])/8,
 193                 struct_offset(struct unw_frame_info, fr_loc[28 - 16])/8,
 194                 struct_offset(struct unw_frame_info, fr_loc[29 - 16])/8,
 195                 struct_offset(struct unw_frame_info, fr_loc[30 - 16])/8,
 196                 struct_offset(struct unw_frame_info, fr_loc[31 - 16])/8,
 197         },
 198         .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 199 #ifdef UNW_DEBUG
 200         .preg_name = {
 201                 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 202                 "r4", "r5", "r6", "r7",
 203                 "ar.unat", "pr", "ar.lc", "ar.fpsr",
 204                 "b1", "b2", "b3", "b4", "b5",
 205                 "f2", "f3", "f4", "f5",
 206                 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 207                 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 208         }
 209 #endif
 210 };
 211
 212 \f
 213 /* Unwind accessors.  */
 214
 215 /*
 216  * Returns offset of rREG in struct pt_regs.
 217  */
 218 static inline unsigned long
 219 pt_regs_off (unsigned long reg)
 220 {
 221         unsigned long off =0;
 222
 223         if (reg >= 1 && reg <= 3)
 224                 off = struct_offset(struct pt_regs, r1) + 8*(reg - 1);
 225         else if (reg <= 11)
 226                 off = struct_offset(struct pt_regs, r8) + 8*(reg - 8);
 227         else if (reg <= 15)
 228                 off = struct_offset(struct pt_regs, r12) + 8*(reg - 12);
 229         else if (reg <= 31)
 230                 off = struct_offset(struct pt_regs, r16) + 8*(reg - 16);
 231         else
 232                 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
 233         return off;
 234 }
 235
 236 static inline struct pt_regs *
 237 get_scratch_regs (struct unw_frame_info *info)
 238 {
 239         if (!info->pt) {
 240                 /* This should not happen with valid unwind info.  */
 241                 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
 242                 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 243                         info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 244                 else
 245                         info->pt = info->sp - 16;
 246         }
 247         UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
 248         return (struct pt_regs *) info->pt;
 249 }
 250
 251 int
 252 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 253 {
 254         unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 255         struct unw_ireg *ireg;
 256         struct pt_regs *pt;
 257
 258         if ((unsigned) regnum - 1 >= 127) {
 259                 if (regnum == 0 && !write) {
 260                         *val = 0;       /* read r0 always returns 0 */
 261                         *nat = 0;
 262                         return 0;
 263                 }
 264                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 265                            __FUNCTION__, regnum);
 266                 return -1;
 267         }
 268
 269         if (regnum < 32) {
 270                 if (regnum >= 4 && regnum <= 7) {
 271                         /* access a preserved register */
 272                         ireg = &info->r4 + (regnum - 4);
 273                         addr = ireg->loc;
 274                         if (addr) {
 275                                 nat_addr = addr + ireg->nat.off;
 276                                 switch (ireg->nat.type) {
 277                                       case UNW_NAT_VAL:
 278                                         /* simulate getf.sig/setf.sig */
 279                                         if (write) {
 280                                                 if (*nat) {
 281                                                         /* write NaTVal and be done with it */
 282                                                         addr[0] = 0;
 283                                                         addr[1] = 0x1fffe;
 284                                                         return 0;
 285                                                 }
 286                                                 addr[1] = 0x1003e;
 287                                         } else {
 288                                                 if (addr[0] == 0 && addr[1] == 0x1ffe) {
 289                                                         /* return NaT and be done with it */
 290                                                         *val = 0;
 291                                                         *nat = 1;
 292                                                         return 0;
 293                                                 }
 294                                         }
 295                                         /* fall through */
 296                                       case UNW_NAT_NONE:
 297                                         dummy_nat = 0;
 298                                         nat_addr = &dummy_nat;
 299                                         break;
 300
 301                                       case UNW_NAT_MEMSTK:
 302                                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 303                                         break;
 304
 305                                       case UNW_NAT_REGSTK:
 306                                         nat_addr = ia64_rse_rnat_addr(addr);
 307                                         if ((unsigned long) addr < info->regstk.limit
 308                                             || (unsigned long) addr >= info->regstk.top)
 309                                         {
 310                                                 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 311                                                         "[0x%lx-0x%lx)\n",
 312                                                         __FUNCTION__, (void *) addr,
 313                                                         info->regstk.limit,
 314                                                         info->regstk.top);
 315                                                 return -1;
 316                                         }
 317                                         if ((unsigned long) nat_addr >= info->regstk.top)
 318                                                 nat_addr = &info->sw->ar_rnat;
 319                                         nat_mask = (1UL << ia64_rse_slot_num(addr));
 320                                         break;
 321                                 }
 322                         } else {
 323                                 addr = &info->sw->r4 + (regnum - 4);
 324                                 nat_addr = &info->sw->ar_unat;
 325                                 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 326                         }
 327                 } else {
 328                         /* access a scratch register */
 329                         pt = get_scratch_regs(info);
 330                         addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 331                         if (info->pri_unat_loc)
 332                                 nat_addr = info->pri_unat_loc;
 333                         else
 334                                 nat_addr = &info->sw->ar_unat;
 335                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 336                 }
 337         } else {
 338                 /* access a stacked register */
 339                 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 340                 nat_addr = ia64_rse_rnat_addr(addr);
 341                 if ((unsigned long) addr < info->regstk.limit
 342                     || (unsigned long) addr >= info->regstk.top)
 343                 {
 344                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 345                                    "of rbs\n",  __FUNCTION__);
 346                         return -1;
 347                 }
 348                 if ((unsigned long) nat_addr >= info->regstk.top)
 349                         nat_addr = &info->sw->ar_rnat;
 350                 nat_mask = (1UL << ia64_rse_slot_num(addr));
 351         }
 352
 353         if (write) {
 354                 *addr = *val;
 355                 if (*nat)
 356                         *nat_addr |= nat_mask;
 357                 else
 358                         *nat_addr &= ~nat_mask;
 359         } else {
 360                 if ((*nat_addr & nat_mask) == 0) {
 361                         *val = *addr;
 362                         *nat = 0;
 363                 } else {
 364                         *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 365                         *nat = 1;
 366                 }
 367         }
 368         return 0;
 369 }
 370
 371 int
 372 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 373 {
 374         unsigned long *addr;
 375         struct pt_regs *pt;
 376
 377         pt = get_scratch_regs(info);
 378         switch (regnum) {
 379                 /* scratch: */
 380               case 0: addr = &pt->b0; break;
 381               case 6: addr = &pt->b6; break;
 382               case 7: addr = &pt->b7; break;
 383
 384                 /* preserved: */
 385               case 1: case 2: case 3: case 4: case 5:
 386                 addr = *(&info->b1_loc + (regnum - 1));
 387                 if (!addr)
 388                         addr = &info->sw->b1 + (regnum - 1);
 389                 break;
 390
 391               default:
 392                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 393                            __FUNCTION__, regnum);
 394                 return -1;
 395         }
 396         if (write)
 397                 *addr = *val;
 398         else
 399                 *val = *addr;
 400         return 0;
 401 }
 402
 403 int
 404 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 405 {
 406         struct ia64_fpreg *addr = 0;
 407         struct pt_regs *pt;
 408
 409         if ((unsigned) (regnum - 2) >= 126) {
 410                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 411                            __FUNCTION__, regnum);
 412                 return -1;
 413         }
 414
 415         pt = get_scratch_regs(info);
 416
 417         if (regnum <= 5) {
 418                 addr = *(&info->f2_loc + (regnum - 2));
 419                 if (!addr)
 420                         addr = &info->sw->f2 + (regnum - 2);
 421         } else if (regnum <= 15) {
 422                 if (regnum <= 9)
 423                         addr = &pt->f6  + (regnum - 6);
 424                 else
 425                         addr = &info->sw->f10 + (regnum - 10);
 426         } else if (regnum <= 31) {
 427                 addr = info->fr_loc[regnum - 16];
 428                 if (!addr)
 429                         addr = &info->sw->f16 + (regnum - 16);
 430         } else {
 431                 struct task_struct *t = info->task;
 432
 433                 if (write)
 434                         ia64_sync_fph(t);
 435                 else
 436                         ia64_flush_fph(t);
 437                 addr = t->thread.fph + (regnum - 32);
 438         }
 439
 440         if (write)
 441                 *addr = *val;
 442         else
 443                 *val = *addr;
 444         return 0;
 445 }
 446
 447 int
 448 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 449 {
 450         unsigned long *addr;
 451         struct pt_regs *pt;
 452
 453         pt = get_scratch_regs(info);
 454         switch (regnum) {
 455               case UNW_AR_BSP:
 456                 addr = info->bsp_loc;
 457                 if (!addr)
 458                         addr = &info->sw->ar_bspstore;
 459                 break;
 460
 461               case UNW_AR_BSPSTORE:
 462                 addr = info->bspstore_loc;
 463                 if (!addr)
 464                         addr = &info->sw->ar_bspstore;
 465                 break;
 466
 467               case UNW_AR_PFS:
 468                 addr = info->pfs_loc;
 469                 if (!addr)
 470                         addr = &info->sw->ar_pfs;
 471                 break;
 472
 473               case UNW_AR_RNAT:
 474                 addr = info->rnat_loc;
 475                 if (!addr)
 476                         addr = &info->sw->ar_rnat;
 477                 break;
 478
 479               case UNW_AR_UNAT:
 480                 addr = info->unat_loc;
 481                 if (!addr)
 482                         addr = &info->sw->ar_unat;
 483                 break;
 484
 485               case UNW_AR_LC:
 486                 addr = info->lc_loc;
 487                 if (!addr)
 488                         addr = &info->sw->ar_lc;
 489                 break;
 490
 491               case UNW_AR_EC:
 492                 if (!info->cfm_loc)
 493                         return -1;
 494                 if (write)
 495                         *info->cfm_loc =
 496                                 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 497                 else
 498                         *val = (*info->cfm_loc >> 52) & 0x3f;
 499                 return 0;
 500
 501               case UNW_AR_FPSR:
 502                 addr = info->fpsr_loc;
 503                 if (!addr)
 504                         addr = &info->sw->ar_fpsr;
 505                 break;
 506
 507               case UNW_AR_RSC:
 508                 addr = &pt->ar_rsc;
 509                 break;
 510
 511               case UNW_AR_CCV:
 512                 addr = &pt->ar_ccv;
 513                 break;
 514
 515               default:
 516                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 517                            __FUNCTION__, regnum);
 518                 return -1;
 519         }
 520
 521         if (write)
 522                 *addr = *val;
 523         else
 524                 *val = *addr;
 525         return 0;
 526 }
 527
 528 int
 529 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 530 {
 531         unsigned long *addr;
 532
 533         addr = info->pr_loc;
 534         if (!addr)
 535                 addr = &info->sw->pr;
 536
 537         if (write)
 538                 *addr = *val;
 539         else
 540                 *val = *addr;
 541         return 0;
 542 }
 543
 544 \f
 545 /* Routines to manipulate the state stack.  */
 546
 547 static inline void
 548 push (struct unw_state_record *sr)
 549 {
 550         struct unw_reg_state *rs;
 551
 552         rs = alloc_reg_state();
 553         if (!rs) {
 554                 printk(KERN_ERR "unwind: cannot stack reg state!\n");
 555                 return;
 556         }
 557         memcpy(rs, &sr->curr, sizeof(*rs));
 558         sr->curr.next = rs;
 559 }
 560
 561 static void
 562 pop (struct unw_state_record *sr)
 563 {
 564         struct unw_reg_state *rs = sr->curr.next;
 565
 566         if (!rs) {
 567                 printk(KERN_ERR "unwind: stack underflow!\n");
 568                 return;
 569         }
 570         memcpy(&sr->curr, rs, sizeof(*rs));
 571         free_reg_state(rs);
 572 }
 573
 574 /* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 575 static struct unw_reg_state *
 576 dup_state_stack (struct unw_reg_state *rs)
 577 {
 578         struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 579
 580         while (rs) {
 581                 copy = alloc_reg_state();
 582                 if (!copy) {
 583                         printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 584                         return NULL;
 585                 }
 586                 memcpy(copy, rs, sizeof(*copy));
 587                 if (first)
 588                         prev->next = copy;
 589                 else
 590                         first = copy;
 591                 rs = rs->next;
 592                 prev = copy;
 593         }
 594         return first;
 595 }
 596
 597 /* Free all stacked register states (but not RS itself).  */
 598 static void
 599 free_state_stack (struct unw_reg_state *rs)
 600 {
 601         struct unw_reg_state *p, *next;
 602
 603         for (p = rs->next; p != NULL; p = next) {
 604                 next = p->next;
 605                 free_reg_state(p);
 606         }
 607         rs->next = NULL;
 608 }
 609 \f
 610 /* Unwind decoder routines */
 611
 612 static enum unw_register_index __attribute__((const))
 613 decode_abreg (unsigned char abreg, int memory)
 614 {
 615         switch (abreg) {
 616               case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 617               case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 618               case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 619               case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 620               case 0x60: return UNW_REG_PR;
 621               case 0x61: return UNW_REG_PSP;
 622               case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 623               case 0x63: return UNW_REG_RP;
 624               case 0x64: return UNW_REG_BSP;
 625               case 0x65: return UNW_REG_BSPSTORE;
 626               case 0x66: return UNW_REG_RNAT;
 627               case 0x67: return UNW_REG_UNAT;
 628               case 0x68: return UNW_REG_FPSR;
 629               case 0x69: return UNW_REG_PFS;
 630               case 0x6a: return UNW_REG_LC;
 631               default:
 632                 break;
 633         }
 634         UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
 635         return UNW_REG_LC;
 636 }
 637
 638 static void
 639 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 640 {
 641         reg->val = val;
 642         reg->where = where;
 643         if (reg->when == UNW_WHEN_NEVER)
 644                 reg->when = when;
 645 }
 646
 647 static void
 648 alloc_spill_area (unsigned long *offp, unsigned long regsize,
 649                   struct unw_reg_info *lo, struct unw_reg_info *hi)
 650 {
 651         struct unw_reg_info *reg;
 652
 653         for (reg = hi; reg >= lo; --reg) {
 654                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 655                         reg->where = UNW_WHERE_PSPREL;
 656                         *offp -= regsize;
 657                         reg->val = *offp;
 658                 }
 659         }
 660 }
 661
 662 static inline void
 663 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 664 {
 665         struct unw_reg_info *reg;
 666
 667         for (reg = *regp; reg <= lim; ++reg) {
 668                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 669                         reg->when = t;
 670                         *regp = reg + 1;
 671                         return;
 672                 }
 673         }
 674         UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __FUNCTION__);
 675 }
 676
 677 static inline void
 678 finish_prologue (struct unw_state_record *sr)
 679 {
 680         struct unw_reg_info *reg;
 681         unsigned long off;
 682         int i;
 683
 684         /*
 685          * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 686          * for Using Unwind Descriptors", rule 3):
 687          */
 688         for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 689                 reg = sr->curr.reg + unw.save_order[i];
 690                 if (reg->where == UNW_WHERE_GR_SAVE) {
 691                         reg->where = UNW_WHERE_GR;
 692                         reg->val = sr->gr_save_loc++;
 693                 }
 694         }
 695
 696         /*
 697          * Next, compute when the fp, general, and branch registers get
 698          * saved.  This must come before alloc_spill_area() because
 699          * we need to know which registers are spilled to their home
 700          * locations.
 701          */
 702         if (sr->imask) {
 703                 unsigned char kind, mask = 0, *cp = sr->imask;
 704                 int t;
 705                 static const unsigned char limit[3] = {
 706                         UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 707                 };
 708                 struct unw_reg_info *(regs[3]);
 709
 710                 regs[0] = sr->curr.reg + UNW_REG_F2;
 711                 regs[1] = sr->curr.reg + UNW_REG_R4;
 712                 regs[2] = sr->curr.reg + UNW_REG_B1;
 713
 714                 for (t = 0; t < sr->region_len; ++t) {
 715                         if ((t & 3) == 0)
 716                                 mask = *cp++;
 717                         kind = (mask >> 2*(3-(t & 3))) & 3;
 718                         if (kind > 0)
 719                                 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 720                                                 sr->region_start + t);
 721                 }
 722         }
 723         /*
 724          * Next, lay out the memory stack spill area:
 725          */
 726         if (sr->any_spills) {
 727                 off = sr->spill_offset;
 728                 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 729                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 730                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 731         }
 732 }
 733
 734 /*
 735  * Region header descriptors.
 736  */
 737
 738 static void
 739 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 740                struct unw_state_record *sr)
 741 {
 742         int i, region_start;
 743
 744         if (!(sr->in_body || sr->first_region))
 745                 finish_prologue(sr);
 746         sr->first_region = 0;
 747
 748         /* check if we're done: */
 749         if (sr->when_target < sr->region_start + sr->region_len) {
 750                 sr->done = 1;
 751                 return;
 752         }
 753
 754         region_start = sr->region_start + sr->region_len;
 755
 756         for (i = 0; i < sr->epilogue_count; ++i)
 757                 pop(sr);
 758         sr->epilogue_count = 0;
 759         sr->epilogue_start = UNW_WHEN_NEVER;
 760
 761         sr->region_start = region_start;
 762         sr->region_len = rlen;
 763         sr->in_body = body;
 764
 765         if (!body) {
 766                 push(sr);
 767
 768                 for (i = 0; i < 4; ++i) {
 769                         if (mask & 0x8)
 770                                 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 771                                         sr->region_start + sr->region_len - 1, grsave++);
 772                         mask <<= 1;
 773                 }
 774                 sr->gr_save_loc = grsave;
 775                 sr->any_spills = 0;
 776                 sr->imask = 0;
 777                 sr->spill_offset = 0x10;        /* default to psp+16 */
 778         }
 779 }
 780
 781 /*
 782  * Prologue descriptors.
 783  */
 784
 785 static inline void
 786 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 787 {
 788         if (abi == 0 && context == 'i') {
 789                 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 790                 UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __FUNCTION__);
 791         }
 792         else
 793                 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 794                                 __FUNCTION__, abi, context);
 795 }
 796
 797 static inline void
 798 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 799 {
 800         int i;
 801
 802         for (i = 0; i < 5; ++i) {
 803                 if (brmask & 1)
 804                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 805                                 sr->region_start + sr->region_len - 1, gr++);
 806                 brmask >>= 1;
 807         }
 808 }
 809
 810 static inline void
 811 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 812 {
 813         int i;
 814
 815         for (i = 0; i < 5; ++i) {
 816                 if (brmask & 1) {
 817                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 818                                 sr->region_start + sr->region_len - 1, 0);
 819                         sr->any_spills = 1;
 820                 }
 821                 brmask >>= 1;
 822         }
 823 }
 824
 825 static inline void
 826 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 827 {
 828         int i;
 829
 830         for (i = 0; i < 4; ++i) {
 831                 if ((grmask & 1) != 0) {
 832                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 833                                 sr->region_start + sr->region_len - 1, 0);
 834                         sr->any_spills = 1;
 835                 }
 836                 grmask >>= 1;
 837         }
 838         for (i = 0; i < 20; ++i) {
 839                 if ((frmask & 1) != 0) {
 840                         int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 841                         set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 842                                 sr->region_start + sr->region_len - 1, 0);
 843                         sr->any_spills = 1;
 844                 }
 845                 frmask >>= 1;
 846         }
 847 }
 848
 849 static inline void
 850 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 851 {
 852         int i;
 853
 854         for (i = 0; i < 4; ++i) {
 855                 if ((frmask & 1) != 0) {
 856                         set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 857                                 sr->region_start + sr->region_len - 1, 0);
 858                         sr->any_spills = 1;
 859                 }
 860                 frmask >>= 1;
 861         }
 862 }
 863
 864 static inline void
 865 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 866 {
 867         int i;
 868
 869         for (i = 0; i < 4; ++i) {
 870                 if ((grmask & 1) != 0)
 871                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 872                                 sr->region_start + sr->region_len - 1, gr++);
 873                 grmask >>= 1;
 874         }
 875 }
 876
 877 static inline void
 878 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 879 {
 880         int i;
 881
 882         for (i = 0; i < 4; ++i) {
 883                 if ((grmask & 1) != 0) {
 884                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 885                                 sr->region_start + sr->region_len - 1, 0);
 886                         sr->any_spills = 1;
 887                 }
 888                 grmask >>= 1;
 889         }
 890 }
 891
 892 static inline void
 893 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 894 {
 895         set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 896                 sr->region_start + MIN((int)t, sr->region_len - 1), 16*size);
 897 }
 898
 899 static inline void
 900 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 901 {
 902         sr->curr.reg[UNW_REG_PSP].when = sr->region_start + MIN((int)t, sr->region_len - 1);
 903 }
 904
 905 static inline void
 906 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 907 {
 908         set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 909 }
 910
 911 static inline void
 912 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 913 {
 914         set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 915                 0x10 - 4*pspoff);
 916 }
 917
 918 static inline void
 919 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 920 {
 921         set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 922                 4*spoff);
 923 }
 924
 925 static inline void
 926 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 927 {
 928         sr->return_link_reg = dst;
 929 }
 930
 931 static inline void
 932 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
 933 {
 934         struct unw_reg_info *reg = sr->curr.reg + regnum;
 935
 936         if (reg->where == UNW_WHERE_NONE)
 937                 reg->where = UNW_WHERE_GR_SAVE;
 938         reg->when = sr->region_start + MIN((int)t, sr->region_len - 1);
 939 }
 940
 941 static inline void
 942 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
 943 {
 944         sr->spill_offset = 0x10 - 4*pspoff;
 945 }
 946
 947 static inline unsigned char *
 948 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
 949 {
 950         sr->imask = imaskp;
 951         return imaskp + (2*sr->region_len + 7)/8;
 952 }
 953
 954 /*
 955  * Body descriptors.
 956  */
 957 static inline void
 958 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
 959 {
 960         sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
 961         sr->epilogue_count = ecount + 1;
 962 }
 963
 964 static inline void
 965 desc_copy_state (unw_word label, struct unw_state_record *sr)
 966 {
 967         struct unw_labeled_state *ls;
 968
 969         for (ls = sr->labeled_states; ls; ls = ls->next) {
 970                 if (ls->label == label) {
 971                         free_state_stack(&sr->curr);
 972                         memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
 973                         sr->curr.next = dup_state_stack(ls->saved_state.next);
 974                         return;
 975                 }
 976         }
 977         printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
 978 }
 979
 980 static inline void
 981 desc_label_state (unw_word label, struct unw_state_record *sr)
 982 {
 983         struct unw_labeled_state *ls;
 984
 985         ls = alloc_labeled_state();
 986         if (!ls) {
 987                 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
 988                 return;
 989         }
 990         ls->label = label;
 991         memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
 992         ls->saved_state.next = dup_state_stack(sr->curr.next);
 993
 994         /* insert into list of labeled states: */
 995         ls->next = sr->labeled_states;
 996         sr->labeled_states = ls;
 997 }
 998
 999 /*
1000  * General descriptors.
1001  */
1002
1003 static inline int
1004 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1005 {
1006         if (sr->when_target <= sr->region_start + MIN((int)t, sr->region_len - 1))
1007                 return 0;
1008         if (qp > 0) {
1009                 if ((sr->pr_val & (1UL << qp)) == 0)
1010                         return 0;
1011                 sr->pr_mask |= (1UL << qp);
1012         }
1013         return 1;
1014 }
1015
1016 static inline void
1017 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1018 {
1019         struct unw_reg_info *r;
1020
1021         if (!desc_is_active(qp, t, sr))
1022                 return;
1023
1024         r = sr->curr.reg + decode_abreg(abreg, 0);
1025         r->where = UNW_WHERE_NONE;
1026         r->when = UNW_WHEN_NEVER;
1027         r->val = 0;
1028 }
1029
1030 static inline void
1031 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1032                      unsigned char ytreg, struct unw_state_record *sr)
1033 {
1034         enum unw_where where = UNW_WHERE_GR;
1035         struct unw_reg_info *r;
1036
1037         if (!desc_is_active(qp, t, sr))
1038                 return;
1039
1040         if (x)
1041                 where = UNW_WHERE_BR;
1042         else if (ytreg & 0x80)
1043                 where = UNW_WHERE_FR;
1044
1045         r = sr->curr.reg + decode_abreg(abreg, 0);
1046         r->where = where;
1047         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1048         r->val = (ytreg & 0x7f);
1049 }
1050
1051 static inline void
1052 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1053                      struct unw_state_record *sr)
1054 {
1055         struct unw_reg_info *r;
1056
1057         if (!desc_is_active(qp, t, sr))
1058                 return;
1059
1060         r = sr->curr.reg + decode_abreg(abreg, 1);
1061         r->where = UNW_WHERE_PSPREL;
1062         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1063         r->val = 0x10 - 4*pspoff;
1064 }
1065
1066 static inline void
1067 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1068                        struct unw_state_record *sr)
1069 {
1070         struct unw_reg_info *r;
1071
1072         if (!desc_is_active(qp, t, sr))
1073                 return;
1074
1075         r = sr->curr.reg + decode_abreg(abreg, 1);
1076         r->where = UNW_WHERE_SPREL;
1077         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1078         r->val = 4*spoff;
1079 }
1080
1081 #define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1082                                                        code);
1083
1084 /*
1085  * region headers:
1086  */
1087 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1088 #define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1089 /*
1090  * prologue descriptors:
1091  */
1092 #define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1093 #define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1094 #define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1095 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1096 #define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1097 #define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1098 #define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1099 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1100 #define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1101 #define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1102 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1103 #define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1104 #define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1105 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1106 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1107 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1108 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1109 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1110 #define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1111 #define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1112 #define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1113 /*
1114  * body descriptors:
1115  */
1116 #define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1117 #define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1118 #define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1119 /*
1120  * general unwind descriptors:
1121  */
1122 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1123 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1124 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1125 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1126 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1127 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1128 #define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1129 #define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1130
1131 #include "unwind_decoder.c"
1132
1133 \f
1134 /* Unwind scripts. */
1135
1136 static inline unw_hash_index_t
1137 hash (unsigned long ip)
1138 {
1139 #       define magic    0x9e3779b97f4a7c16      /* based on (sqrt(5)/2-1)*2^64 */
1140
1141         return (ip >> 4)*magic >> (64 - UNW_LOG_HASH_SIZE);
1142 }
1143
1144 static inline long
1145 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1146 {
1147         read_lock(&script->lock);
1148         if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1149                 /* keep the read lock... */
1150                 return 1;
1151         read_unlock(&script->lock);
1152         return 0;
1153 }
1154
1155 static inline struct unw_script *
1156 script_lookup (struct unw_frame_info *info)
1157 {
1158         struct unw_script *script = unw.cache + info->hint;
1159         unsigned short index;
1160         unsigned long ip, pr;
1161
1162         if (UNW_DEBUG_ON(0))
1163                 return 0;       /* Always regenerate scripts in debug mode */
1164
1165         STAT(++unw.stat.cache.lookups);
1166
1167         ip = info->ip;
1168         pr = info->pr;
1169
1170         if (cache_match(script, ip, pr)) {
1171                 STAT(++unw.stat.cache.hinted_hits);
1172                 return script;
1173         }
1174
1175         index = unw.hash[hash(ip)];
1176         if (index >= UNW_CACHE_SIZE)
1177                 return 0;
1178
1179         script = unw.cache + index;
1180         while (1) {
1181                 if (cache_match(script, ip, pr)) {
1182                         /* update hint; no locking required as single-word writes are atomic */
1183                         STAT(++unw.stat.cache.normal_hits);
1184                         unw.cache[info->prev_script].hint = script - unw.cache;
1185                         return script;
1186                 }
1187                 if (script->coll_chain >= UNW_HASH_SIZE)
1188                         return 0;
1189                 script = unw.cache + script->coll_chain;
1190                 STAT(++unw.stat.cache.collision_chain_traversals);
1191         }
1192 }
1193
1194 /*
1195  * On returning, a write lock for the SCRIPT is still being held.
1196  */
1197 static inline struct unw_script *
1198 script_new (unsigned long ip)
1199 {
1200         struct unw_script *script, *prev, *tmp;
1201         unw_hash_index_t index;
1202         unsigned long flags;
1203         unsigned short head;
1204
1205         STAT(++unw.stat.script.news);
1206
1207         /*
1208          * Can't (easily) use cmpxchg() here because of ABA problem
1209          * that is intrinsic in cmpxchg()...
1210          */
1211         spin_lock_irqsave(&unw.lock, flags);
1212         {
1213                 head = unw.lru_head;
1214                 script = unw.cache + head;
1215                 unw.lru_head = script->lru_chain;
1216         }
1217         spin_unlock(&unw.lock);
1218
1219         /*
1220          * We'd deadlock here if we interrupted a thread that is holding a read lock on
1221          * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1222          * alternative would be to disable interrupts whenever we hold a read-lock, but
1223          * that seems silly.
1224          */
1225         if (!write_trylock(&script->lock))
1226                 return NULL;
1227
1228         spin_lock(&unw.lock);
1229         {
1230                 /* re-insert script at the tail of the LRU chain: */
1231                 unw.cache[unw.lru_tail].lru_chain = head;
1232                 unw.lru_tail = head;
1233
1234                 /* remove the old script from the hash table (if it's there): */
1235                 if (script->ip) {
1236                         index = hash(script->ip);
1237                         tmp = unw.cache + unw.hash[index];
1238                         prev = 0;
1239                         while (1) {
1240                                 if (tmp == script) {
1241                                         if (prev)
1242                                                 prev->coll_chain = tmp->coll_chain;
1243                                         else
1244                                                 unw.hash[index] = tmp->coll_chain;
1245                                         break;
1246                                 } else
1247                                         prev = tmp;
1248                                 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1249                                 /* old script wasn't in the hash-table */
1250                                         break;
1251                                 tmp = unw.cache + tmp->coll_chain;
1252                         }
1253                 }
1254
1255                 /* enter new script in the hash table */
1256                 index = hash(ip);
1257                 script->coll_chain = unw.hash[index];
1258                 unw.hash[index] = script - unw.cache;
1259
1260                 script->ip = ip;        /* set new IP while we're holding the locks */
1261
1262                 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1263         }
1264         spin_unlock_irqrestore(&unw.lock, flags);
1265
1266         script->flags = 0;
1267         script->hint = 0;
1268         script->count = 0;
1269         return script;
1270 }
1271
1272 static void
1273 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1274 {
1275         script->pr_mask = sr->pr_mask;
1276         script->pr_val = sr->pr_val;
1277         /*
1278          * We could down-grade our write-lock on script->lock here but
1279          * the rwlock API doesn't offer atomic lock downgrading, so
1280          * we'll just keep the write-lock and release it later when
1281          * we're done using the script.
1282          */
1283 }
1284
1285 static inline void
1286 script_emit (struct unw_script *script, struct unw_insn insn)
1287 {
1288         if (script->count >= UNW_MAX_SCRIPT_LEN) {
1289                 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1290                         __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1291                 return;
1292         }
1293         script->insn[script->count++] = insn;
1294 }
1295
1296 static inline void
1297 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1298 {
1299         struct unw_reg_info *r = sr->curr.reg + i;
1300         enum unw_insn_opcode opc;
1301         struct unw_insn insn;
1302         unsigned long val = 0;
1303
1304         switch (r->where) {
1305               case UNW_WHERE_GR:
1306                 if (r->val >= 32) {
1307                         /* register got spilled to a stacked register */
1308                         opc = UNW_INSN_SETNAT_TYPE;
1309                         val = UNW_NAT_REGSTK;
1310                 } else
1311                         /* register got spilled to a scratch register */
1312                         opc = UNW_INSN_SETNAT_MEMSTK;
1313                 break;
1314
1315               case UNW_WHERE_FR:
1316                 opc = UNW_INSN_SETNAT_TYPE;
1317                 val = UNW_NAT_VAL;
1318                 break;
1319
1320               case UNW_WHERE_BR:
1321                 opc = UNW_INSN_SETNAT_TYPE;
1322                 val = UNW_NAT_NONE;
1323                 break;
1324
1325               case UNW_WHERE_PSPREL:
1326               case UNW_WHERE_SPREL:
1327                 opc = UNW_INSN_SETNAT_MEMSTK;
1328                 break;
1329
1330               default:
1331                 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1332                            __FUNCTION__, r->where);
1333                 return;
1334         }
1335         insn.opc = opc;
1336         insn.dst = unw.preg_index[i];
1337         insn.val = val;
1338         script_emit(script, insn);
1339 }
1340
1341 static void
1342 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1343 {
1344         struct unw_reg_info *r = sr->curr.reg + i;
1345         enum unw_insn_opcode opc;
1346         unsigned long val, rval;
1347         struct unw_insn insn;
1348         long need_nat_info;
1349
1350         if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1351                 return;
1352
1353         opc = UNW_INSN_MOVE;
1354         val = rval = r->val;
1355         need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1356
1357         switch (r->where) {
1358               case UNW_WHERE_GR:
1359                 if (rval >= 32) {
1360                         opc = UNW_INSN_MOVE_STACKED;
1361                         val = rval - 32;
1362                 } else if (rval >= 4 && rval <= 7) {
1363                         if (need_nat_info) {
1364                                 opc = UNW_INSN_MOVE2;
1365                                 need_nat_info = 0;
1366                         }
1367                         val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1368                 } else {
1369                         /* register got spilled to a scratch register */
1370                         opc = UNW_INSN_MOVE_SCRATCH;
1371                         val = pt_regs_off(rval);
1372                 }
1373                 break;
1374
1375               case UNW_WHERE_FR:
1376                 if (rval <= 5)
1377                         val = unw.preg_index[UNW_REG_F2  + (rval -  1)];
1378                 else if (rval >= 16 && rval <= 31)
1379                         val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1380                 else {
1381                         opc = UNW_INSN_MOVE_SCRATCH;
1382                         if (rval <= 9)
1383                                 val = struct_offset(struct pt_regs, f6) + 16*(rval - 6);
1384                         else
1385                                 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1386                                            __FUNCTION__, rval);
1387                 }
1388                 break;
1389
1390               case UNW_WHERE_BR:
1391                 if (rval >= 1 && rval <= 5)
1392                         val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1393                 else {
1394                         opc = UNW_INSN_MOVE_SCRATCH;
1395                         if (rval == 0)
1396                                 val = struct_offset(struct pt_regs, b0);
1397                         else if (rval == 6)
1398                                 val = struct_offset(struct pt_regs, b6);
1399                         else
1400                                 val = struct_offset(struct pt_regs, b7);
1401                 }
1402                 break;
1403
1404               case UNW_WHERE_SPREL:
1405                 opc = UNW_INSN_ADD_SP;
1406                 break;
1407
1408               case UNW_WHERE_PSPREL:
1409                 opc = UNW_INSN_ADD_PSP;
1410                 break;
1411
1412               default:
1413                 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1414                            __FUNCTION__, i, r->where);
1415                 break;
1416         }
1417         insn.opc = opc;
1418         insn.dst = unw.preg_index[i];
1419         insn.val = val;
1420         script_emit(script, insn);
1421         if (need_nat_info)
1422                 emit_nat_info(sr, i, script);
1423
1424         if (i == UNW_REG_PSP) {
1425                 /*
1426                  * info->psp must contain the _value_ of the previous
1427                  * sp, not it's save location.  We get this by
1428                  * dereferencing the value we just stored in
1429                  * info->psp:
1430                  */
1431                 insn.opc = UNW_INSN_LOAD;
1432                 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1433                 script_emit(script, insn);
1434         }
1435 }
1436
1437 static inline const struct unw_table_entry *
1438 lookup (struct unw_table *table, unsigned long rel_ip)
1439 {
1440         const struct unw_table_entry *e = 0;
1441         unsigned long lo, hi, mid;
1442
1443         /* do a binary search for right entry: */
1444         for (lo = 0, hi = table->length; lo < hi; ) {
1445                 mid = (lo + hi) / 2;
1446                 e = &table->array[mid];
1447                 if (rel_ip < e->start_offset)
1448                         hi = mid;
1449                 else if (rel_ip >= e->end_offset)
1450                         lo = mid + 1;
1451                 else
1452                         break;
1453         }
1454         if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1455                 return NULL;
1456         return e;
1457 }
1458
1459 /*
1460  * Build an unwind script that unwinds from state OLD_STATE to the
1461  * entrypoint of the function that called OLD_STATE.
1462  */
1463 static inline struct unw_script *
1464 build_script (struct unw_frame_info *info)
1465 {
1466         const struct unw_table_entry *e = 0;
1467         struct unw_script *script = 0;
1468         struct unw_labeled_state *ls, *next;
1469         unsigned long ip = info->ip;
1470         struct unw_state_record sr;
1471         struct unw_table *table;
1472         struct unw_reg_info *r;
1473         struct unw_insn insn;
1474         u8 *dp, *desc_end;
1475         u64 hdr;
1476         int i;
1477         STAT(unsigned long start, parse_start;)
1478
1479         STAT(++unw.stat.script.builds; start = ia64_get_itc());
1480
1481         /* build state record */
1482         memset(&sr, 0, sizeof(sr));
1483         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1484                 r->when = UNW_WHEN_NEVER;
1485         sr.pr_val = info->pr;
1486
1487         UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1488         script = script_new(ip);
1489         if (!script) {
1490                 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __FUNCTION__);
1491                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1492                 return 0;
1493         }
1494         unw.cache[info->prev_script].hint = script - unw.cache;
1495
1496         /* search the kernels and the modules' unwind tables for IP: */
1497
1498         STAT(parse_start = ia64_get_itc());
1499
1500         for (table = unw.tables; table; table = table->next) {
1501                 if (ip >= table->start && ip < table->end) {
1502                         e = lookup(table, ip - table->segment_base);
1503                         break;
1504                 }
1505         }
1506         if (!e) {
1507                 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1508                 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1509                         __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1510                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1511                 sr.curr.reg[UNW_REG_RP].when = -1;
1512                 sr.curr.reg[UNW_REG_RP].val = 0;
1513                 compile_reg(&sr, UNW_REG_RP, script);
1514                 script_finalize(script, &sr);
1515                 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1516                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1517                 return script;
1518         }
1519
1520         sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1521                           + (ip & 0xfUL));
1522         hdr = *(u64 *) (table->segment_base + e->info_offset);
1523         dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1524         desc_end = dp + 8*UNW_LENGTH(hdr);
1525
1526         while (!sr.done && dp < desc_end)
1527                 dp = unw_decode(dp, sr.in_body, &sr);
1528
1529         if (sr.when_target > sr.epilogue_start) {
1530                 /*
1531                  * sp has been restored and all values on the memory stack below
1532                  * psp also have been restored.
1533                  */
1534                 sr.curr.reg[UNW_REG_PSP].val = 0;
1535                 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1536                 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1537                 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1538                         if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1539                             || r->where == UNW_WHERE_SPREL)
1540                         {
1541                                 r->val = 0;
1542                                 r->where = UNW_WHERE_NONE;
1543                                 r->when = UNW_WHEN_NEVER;
1544                         }
1545         }
1546
1547         script->flags = sr.flags;
1548
1549         /*
1550          * If RP did't get saved, generate entry for the return link
1551          * register.
1552          */
1553         if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1554                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1555                 sr.curr.reg[UNW_REG_RP].when = -1;
1556                 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1557                 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1558                            __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1559                            sr.curr.reg[UNW_REG_RP].val);
1560         }
1561
1562 #ifdef UNW_DEBUG
1563         UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1564                 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1565         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1566                 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1567                         UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1568                         switch (r->where) {
1569                               case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1570                               case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1571                               case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1572                               case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1573                               case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1574                               case UNW_WHERE_NONE:
1575                                 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1576                                 break;
1577
1578                               default:
1579                                 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1580                                 break;
1581                         }
1582                         UNW_DPRINT(1, "\t\t%d\n", r->when);
1583                 }
1584         }
1585 #endif
1586
1587         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1588
1589         /* translate state record into unwinder instructions: */
1590
1591         /*
1592          * First, set psp if we're dealing with a fixed-size frame;
1593          * subsequent instructions may depend on this value.
1594          */
1595         if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1596             && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1597             && sr.curr.reg[UNW_REG_PSP].val != 0) {
1598                 /* new psp is sp plus frame size */
1599                 insn.opc = UNW_INSN_ADD;
1600                 insn.dst = struct_offset(struct unw_frame_info, psp)/8;
1601                 insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1602                 script_emit(script, insn);
1603         }
1604
1605         /* determine where the primary UNaT is: */
1606         if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1607                 i = UNW_REG_PRI_UNAT_MEM;
1608         else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1609                 i = UNW_REG_PRI_UNAT_GR;
1610         else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1611                 i = UNW_REG_PRI_UNAT_MEM;
1612         else
1613                 i = UNW_REG_PRI_UNAT_GR;
1614
1615         compile_reg(&sr, i, script);
1616
1617         for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1618                 compile_reg(&sr, i, script);
1619
1620         /* free labeled register states & stack: */
1621
1622         STAT(parse_start = ia64_get_itc());
1623         for (ls = sr.labeled_states; ls; ls = next) {
1624                 next = ls->next;
1625                 free_state_stack(&ls->saved_state);
1626                 free_labeled_state(ls);
1627         }
1628         free_state_stack(&sr.curr);
1629         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1630
1631         script_finalize(script, &sr);
1632         STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1633         return script;
1634 }
1635
1636 /*
1637  * Apply the unwinding actions represented by OPS and update SR to
1638  * reflect the state that existed upon entry to the function that this
1639  * unwinder represents.
1640  */
1641 static inline void
1642 run_script (struct unw_script *script, struct unw_frame_info *state)
1643 {
1644         struct unw_insn *ip, *limit, next_insn;
1645         unsigned long opc, dst, val, off;
1646         unsigned long *s = (unsigned long *) state;
1647         STAT(unsigned long start;)
1648
1649         STAT(++unw.stat.script.runs; start = ia64_get_itc());
1650         state->flags = script->flags;
1651         ip = script->insn;
1652         limit = script->insn + script->count;
1653         next_insn = *ip;
1654
1655         while (ip++ < limit) {
1656                 opc = next_insn.opc;
1657                 dst = next_insn.dst;
1658                 val = next_insn.val;
1659                 next_insn = *ip;
1660
1661           redo:
1662                 switch (opc) {
1663                       case UNW_INSN_ADD:
1664                         s[dst] += val;
1665                         break;
1666
1667                       case UNW_INSN_MOVE2:
1668                         if (!s[val])
1669                                 goto lazy_init;
1670                         s[dst+1] = s[val+1];
1671                         s[dst] = s[val];
1672                         break;
1673
1674                       case UNW_INSN_MOVE:
1675                         if (!s[val])
1676                                 goto lazy_init;
1677                         s[dst] = s[val];
1678                         break;
1679
1680                       case UNW_INSN_MOVE_SCRATCH:
1681                         if (state->pt) {
1682                                 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1683                         } else {
1684                                 s[dst] = 0;
1685                                 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1686                                            __FUNCTION__, dst, val);
1687                         }
1688                         break;
1689
1690                       case UNW_INSN_MOVE_STACKED:
1691                         s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1692                                                                     val);
1693                         break;
1694
1695                       case UNW_INSN_ADD_PSP:
1696                         s[dst] = state->psp + val;
1697                         break;
1698
1699                       case UNW_INSN_ADD_SP:
1700                         s[dst] = state->sp + val;
1701                         break;
1702
1703                       case UNW_INSN_SETNAT_MEMSTK:
1704                         if (!state->pri_unat_loc)
1705                                 state->pri_unat_loc = &state->sw->ar_unat;
1706                         /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1707                         s[dst+1] = (*state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1708                         break;
1709
1710                       case UNW_INSN_SETNAT_TYPE:
1711                         s[dst+1] = val;
1712                         break;
1713
1714                       case UNW_INSN_LOAD:
1715 #ifdef UNW_DEBUG
1716                         if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1717                             || s[val] < TASK_SIZE)
1718                         {
1719                                 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1720                                            __FUNCTION__, s[val]);
1721                                 break;
1722                         }
1723 #endif
1724                         s[dst] = *(unsigned long *) s[val];
1725                         break;
1726                 }
1727         }
1728         STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1729         return;
1730
1731   lazy_init:
1732         off = unw.sw_off[val];
1733         s[val] = (unsigned long) state->sw + off;
1734         if (off >= struct_offset(struct switch_stack, r4)
1735             && off <= struct_offset(struct switch_stack, r7))
1736                 /*
1737                  * We're initializing a general register: init NaT info, too.  Note that
1738                  * the offset is a multiple of 8 which gives us the 3 bits needed for
1739                  * the type field.
1740                  */
1741                 s[val+1] = (struct_offset(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1742         goto redo;
1743 }
1744
1745 static int
1746 find_save_locs (struct unw_frame_info *info)
1747 {
1748         int have_write_lock = 0;
1749         struct unw_script *scr;
1750
1751         if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1752                 /* don't let obviously bad addresses pollute the cache */
1753                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1754                 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1755                 info->rp_loc = 0;
1756                 return -1;
1757         }
1758
1759         scr = script_lookup(info);
1760         if (!scr) {
1761                 scr = build_script(info);
1762                 if (!scr) {
1763                         UNW_DPRINT(0,
1764                                    "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1765                                    __FUNCTION__, info->ip);
1766                         return -1;
1767                 }
1768                 have_write_lock = 1;
1769         }
1770         info->hint = scr->hint;
1771         info->prev_script = scr - unw.cache;
1772
1773         run_script(scr, info);
1774
1775         if (have_write_lock)
1776                 write_unlock(&scr->lock);
1777         else
1778                 read_unlock(&scr->lock);
1779         return 0;
1780 }
1781
1782 int
1783 unw_unwind (struct unw_frame_info *info)
1784 {
1785         unsigned long prev_ip, prev_sp, prev_bsp;
1786         unsigned long ip, pr, num_regs;
1787         STAT(unsigned long start, flags;)
1788         int retval;
1789
1790         STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1791
1792         prev_ip = info->ip;
1793         prev_sp = info->sp;
1794         prev_bsp = info->bsp;
1795
1796         /* restore the ip */
1797         if (!info->rp_loc) {
1798                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1799                 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1800                            __FUNCTION__, info->ip);
1801                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1802                 return -1;
1803         }
1804         ip = info->ip = *info->rp_loc;
1805         if (ip < GATE_ADDR + PAGE_SIZE) {
1806                 /*
1807                  * We don't have unwind info for the gate page, so we consider that part
1808                  * of user-space for the purpose of unwinding.
1809                  */
1810                 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1811                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1812                 return -1;
1813         }
1814
1815         /* restore the cfm: */
1816         if (!info->pfs_loc) {
1817                 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1818                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1819                 return -1;
1820         }
1821         info->cfm_loc = info->pfs_loc;
1822
1823         /* restore the bsp: */
1824         pr = info->pr;
1825         num_regs = 0;
1826         if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1827                 info->pt = info->sp + 16;
1828                 if ((pr & (1UL << pNonSys)) != 0)
1829                         num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1830                 info->pfs_loc =
1831                         (unsigned long *) (info->pt + struct_offset(struct pt_regs, ar_pfs));
1832                 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1833         } else
1834                 num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1835         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1836         if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1837                 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1838                         __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1839                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1840                 return -1;
1841         }
1842
1843         /* restore the sp: */
1844         info->sp = info->psp;
1845         if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1846                 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1847                         __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1848                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1849                 return -1;
1850         }
1851
1852         if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1853                 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1854                            __FUNCTION__, ip);
1855                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1856                 return -1;
1857         }
1858
1859         /* as we unwind, the saved ar.unat becomes the primary unat: */
1860         info->pri_unat_loc = info->unat_loc;
1861
1862         /* finally, restore the predicates: */
1863         unw_get_pr(info, &info->pr);
1864
1865         retval = find_save_locs(info);
1866         STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1867         return retval;
1868 }
1869
1870 int
1871 unw_unwind_to_user (struct unw_frame_info *info)
1872 {
1873         unsigned long ip;
1874
1875         while (unw_unwind(info) >= 0) {
1876                 if (unw_get_rp(info, &ip) < 0) {
1877                         unw_get_ip(info, &ip);
1878                         UNW_DPRINT(0, "unwind.%s: failed to read return pointer (ip=0x%lx)\n",
1879                                    __FUNCTION__, ip);
1880                         return -1;
1881                 }
1882                 /*
1883                  * We don't have unwind info for the gate page, so we consider that part
1884                  * of user-space for the purpose of unwinding.
1885                  */
1886                 if (ip < GATE_ADDR + PAGE_SIZE)
1887                         return 0;
1888         }
1889         unw_get_ip(info, &ip);
1890         UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n", __FUNCTION__, ip);
1891         return -1;
1892 }
1893
1894 static void
1895 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1896                  struct switch_stack *sw, unsigned long stktop)
1897 {
1898         unsigned long rbslimit, rbstop, stklimit;
1899         STAT(unsigned long start, flags;)
1900
1901         STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1902
1903         /*
1904          * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1905          * don't want to do that because it would be slow as each preserved register would
1906          * have to be processed.  Instead, what we do here is zero out the frame info and
1907          * start the unwind process at the function that created the switch_stack frame.
1908          * When a preserved value in switch_stack needs to be accessed, run_script() will
1909          * initialize the appropriate pointer on demand.
1910          */
1911         memset(info, 0, sizeof(*info));
1912
1913         rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1914         rbstop   = sw->ar_bspstore;
1915         if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1916                 rbstop = rbslimit;
1917
1918         stklimit = (unsigned long) t + IA64_STK_OFFSET;
1919         if (stktop <= rbstop)
1920                 stktop = rbstop;
1921
1922         info->regstk.limit = rbslimit;
1923         info->regstk.top   = rbstop;
1924         info->memstk.limit = stklimit;
1925         info->memstk.top   = stktop;
1926         info->task = t;
1927         info->sw  = sw;
1928         info->sp = info->psp = stktop;
1929         info->pr = sw->pr;
1930         UNW_DPRINT(3, "unwind.%s:\n"
1931                    "  task   0x%lx\n"
1932                    "  rbs = [0x%lx-0x%lx)\n"
1933                    "  stk = [0x%lx-0x%lx)\n"
1934                    "  pr     0x%lx\n"
1935                    "  sw     0x%lx\n"
1936                    "  sp     0x%lx\n",
1937                    __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
1938                    info->pr, (unsigned long) info->sw, info->sp);
1939         STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
1940 }
1941
1942 void
1943 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
1944                             struct pt_regs *pt, struct switch_stack *sw)
1945 {
1946         unsigned long sof;
1947
1948         init_frame_info(info, t, sw, pt->r12);
1949         info->cfm_loc = &pt->cr_ifs;
1950         info->unat_loc = &pt->ar_unat;
1951         info->pfs_loc = &pt->ar_pfs;
1952         sof = *info->cfm_loc & 0x7f;
1953         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
1954         info->ip = pt->cr_iip + ia64_psr(pt)->ri;
1955         info->pt = (unsigned long) pt;
1956         UNW_DPRINT(3, "unwind.%s:\n"
1957                    "  bsp    0x%lx\n"
1958                    "  sof    0x%lx\n"
1959                    "  ip     0x%lx\n",
1960                    __FUNCTION__, info->bsp, sof, info->ip);
1961         find_save_locs(info);
1962 }
1963
1964 void
1965 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
1966 {
1967         unsigned long sol;
1968
1969         init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
1970         info->cfm_loc = &sw->ar_pfs;
1971         sol = (*info->cfm_loc >> 7) & 0x7f;
1972         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
1973         info->ip = sw->b0;
1974         UNW_DPRINT(3, "unwind.%s:\n"
1975                    "  bsp    0x%lx\n"
1976                    "  sol    0x%lx\n"
1977                    "  ip     0x%lx\n",
1978                    __FUNCTION__, info->bsp, sol, info->ip);
1979         find_save_locs(info);
1980 }
1981
1982 void
1983 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
1984 {
1985         struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
1986
1987         UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
1988         unw_init_frame_info(info, t, sw);
1989 }
1990
1991 static void
1992 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
1993                    unsigned long gp, const void *table_start, const void *table_end)
1994 {
1995         const struct unw_table_entry *start = table_start, *end = table_end;
1996
1997         table->name = name;
1998         table->segment_base = segment_base;
1999         table->gp = gp;
2000         table->start = segment_base + start[0].start_offset;
2001         table->end = segment_base + end[-1].end_offset;
2002         table->array = start;
2003         table->length = end - start;
2004 }
2005
2006 void *
2007 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2008                       const void *table_start, const void *table_end)
2009 {
2010         const struct unw_table_entry *start = table_start, *end = table_end;
2011         struct unw_table *table;
2012         unsigned long flags;
2013
2014         if (end - start <= 0) {
2015                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2016                            __FUNCTION__);
2017                 return 0;
2018         }
2019
2020         table = kmalloc(sizeof(*table), GFP_USER);
2021         if (!table)
2022                 return 0;
2023
2024         init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2025
2026         spin_lock_irqsave(&unw.lock, flags);
2027         {
2028                 /* keep kernel unwind table at the front (it's searched most commonly): */
2029                 table->next = unw.tables->next;
2030                 unw.tables->next = table;
2031         }
2032         spin_unlock_irqrestore(&unw.lock, flags);
2033
2034         return table;
2035 }
2036
2037 void
2038 unw_remove_unwind_table (void *handle)
2039 {
2040         struct unw_table *table, *prev;
2041         struct unw_script *tmp;
2042         unsigned long flags;
2043         long index;
2044
2045         if (!handle) {
2046                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2047                            __FUNCTION__);
2048                 return;
2049         }
2050
2051         table = handle;
2052         if (table == &unw.kernel_table) {
2053                 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2054                            "no-can-do!\n", __FUNCTION__);
2055                 return;
2056         }
2057
2058         spin_lock_irqsave(&unw.lock, flags);
2059         {
2060                 /* first, delete the table: */
2061
2062                 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2063                         if (prev->next == table)
2064                                 break;
2065                 if (!prev) {
2066                         UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2067                                    __FUNCTION__, (void *) table);
2068                         spin_unlock_irqrestore(&unw.lock, flags);
2069                         return;
2070                 }
2071                 prev->next = table->next;
2072         }
2073         spin_unlock_irqrestore(&unw.lock, flags);
2074
2075         /* next, remove hash table entries for this table */
2076
2077         for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2078                 tmp = unw.cache + unw.hash[index];
2079                 if (unw.hash[index] >= UNW_CACHE_SIZE
2080                     || tmp->ip < table->start || tmp->ip >= table->end)
2081                         continue;
2082
2083                 write_lock(&tmp->lock);
2084                 {
2085                         if (tmp->ip >= table->start && tmp->ip < table->end) {
2086                                 unw.hash[index] = tmp->coll_chain;
2087                                 tmp->ip = 0;
2088                         }
2089                 }
2090                 write_unlock(&tmp->lock);
2091         }
2092
2093         kfree(table);
2094 }
2095
2096 void
2097 unw_create_gate_table (void)
2098 {
2099         extern char __start_gate_section[], __stop_gate_section[];
2100         unsigned long *lp, start, end, segbase = unw.kernel_table.segment_base;
2101         const struct unw_table_entry *entry, *first, *unw_table_end;
2102         extern int ia64_unw_end;
2103         size_t info_size, size;
2104         char *info;
2105
2106         start = (unsigned long) __start_gate_section - segbase;
2107         end   = (unsigned long) __stop_gate_section - segbase;
2108         unw_table_end = (struct unw_table_entry *) &ia64_unw_end;
2109         size  = 0;
2110         first = lookup(&unw.kernel_table, start);
2111
2112         for (entry = first; entry < unw_table_end && entry->start_offset < end; ++entry)
2113                 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2114         size += 8;      /* reserve space for "end of table" marker */
2115
2116         unw.gate_table = alloc_bootmem(size);
2117         if (!unw.gate_table) {
2118                 unw.gate_table_size = 0;
2119                 printk(KERN_ERR "unwind: unable to create unwind data for gate page!\n");
2120                 return;
2121         }
2122         unw.gate_table_size = size;
2123
2124         lp = unw.gate_table;
2125         info = (char *) unw.gate_table + size;
2126
2127         for (entry = first; entry < unw_table_end && entry->start_offset < end; ++entry, lp += 3) {
2128                 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2129                 info -= info_size;
2130                 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2131
2132                 lp[0] = entry->start_offset - start + GATE_ADDR;        /* start */
2133                 lp[1] = entry->end_offset - start + GATE_ADDR;          /* end */
2134                 lp[2] = info - (char *) unw.gate_table;                 /* info */
2135         }
2136         *lp = 0;        /* end-of-table marker */
2137 }
2138
2139 void
2140 unw_init (void)
2141 {
2142         extern int ia64_unw_start, ia64_unw_end, __gp;
2143         extern void unw_hash_index_t_is_too_narrow (void);
2144         long i, off;
2145
2146         if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2147                 unw_hash_index_t_is_too_narrow();
2148
2149         unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2150         unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2151         unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2152         unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2153         unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2154         unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2155         unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2156         unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2157         for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2158                 unw.sw_off[unw.preg_index[i]] = off;
2159         for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2160                 unw.sw_off[unw.preg_index[i]] = off;
2161         for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2162                 unw.sw_off[unw.preg_index[i]] = off;
2163         for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2164                 unw.sw_off[unw.preg_index[i]] = off;
2165
2166         for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2167                 if (i > 0)
2168                         unw.cache[i].lru_chain = (i - 1);
2169                 unw.cache[i].coll_chain = -1;
2170                 unw.cache[i].lock = RW_LOCK_UNLOCKED;
2171         }
2172         unw.lru_head = UNW_CACHE_SIZE - 1;
2173         unw.lru_tail = 0;
2174
2175         init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) &__gp,
2176                           &ia64_unw_start, &ia64_unw_end);
2177 }
2178
2179 /*
2180  * This system call copies the unwind data into the buffer pointed to by BUF and returns
2181  * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2182  * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2183  * unwind data.
2184  *
2185  * The first portion of the unwind data contains an unwind table and rest contains the
2186  * associated unwind info (in no particular order).  The unwind table consists of a table
2187  * of entries of the form:
2188  *
2189  *      u64 start;      (64-bit address of start of function)
2190  *      u64 end;        (64-bit address of start of function)
2191  *      u64 info;       (BUF-relative offset to unwind info)
2192  *
2193  * The end of the unwind table is indicated by an entry with a START address of zero.
2194  *
2195  * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2196  * on the format of the unwind info.
2197  *
2198  * ERRORS
2199  *      EFAULT  BUF points outside your accessible address space.
2200  */
2201 asmlinkage long
2202 sys_getunwind (void *buf, size_t buf_size)
2203 {
2204         if (buf && buf_size >= unw.gate_table_size)
2205                 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2206                         return -EFAULT;
2207         return unw.gate_table_size;
2208 }