arch/ia64/kernel/unwind.c

   1 /*
   2  * Copyright (C) 1999-2003 Hewlett-Packard Co
   3  *      David Mosberger-Tang <davidm@hpl.hp.com>
   4  * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
   5  *      - Change pt_regs_off() to make it less dependant on pt_regs structure.
   6  */
   7 /*
   8  * This file implements call frame unwind support for the Linux
   9  * kernel.  Parsing and processing the unwind information is
  10  * time-consuming, so this implementation translates the unwind
  11  * descriptors into unwind scripts.  These scripts are very simple
  12  * (basically a sequence of assignments) and efficient to execute.
  13  * They are cached for later re-use.  Each script is specific for a
  14  * given instruction pointer address and the set of predicate values
  15  * that the script depends on (most unwind descriptors are
  16  * unconditional and scripts often do not depend on predicates at
  17  * all).  This code is based on the unwind conventions described in
  18  * the "IA-64 Software Conventions and Runtime Architecture" manual.
  19  *
  20  * SMP conventions:
  21  *      o updates to the global unwind data (in structure "unw") are serialized
  22  *        by the unw.lock spinlock
  23  *      o each unwind script has its own read-write lock; a thread must acquire
  24  *        a read lock before executing a script and must acquire a write lock
  25  *        before modifying a script
  26  *      o if both the unw.lock spinlock and a script's read-write lock must be
  27  *        acquired, then the read-write lock must be acquired first.
  28  */
  29 #include <linux/bootmem.h>
  30 #include <linux/elf.h>
  31 #include <linux/kernel.h>
  32 #include <linux/sched.h>
  33 #include <linux/slab.h>
  34
  35 #include <asm/unwind.h>
  36
  37 #include <asm/delay.h>
  38 #include <asm/page.h>
  39 #include <asm/ptrace.h>
  40 #include <asm/ptrace_offsets.h>
  41 #include <asm/rse.h>
  42 #include <asm/system.h>
  43 #include <asm/uaccess.h>
  44
  45 #include "entry.h"
  46 #include "unwind_i.h"
  47
  48 #define MIN(a,b)        ((a) < (b) ? (a) : (b))
  49 #define p5              5
  50
  51 #define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  52 #define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  53
  54 #define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  55 #define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  56
  57 #define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  58
  59 #ifdef UNW_DEBUG
  60   static unsigned int unw_debug_level = UNW_DEBUG;
  61 #  ifdef CONFIG_KDB
  62 #    include <linux/kdb.h>
  63 #    define UNW_DEBUG_ON(n)     (unw_debug_level >= n && !KDB_IS_RUNNING())
  64 #    define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) kdb_printf(__VA_ARGS__)
  65 #  else /* !CONFIG_KDB */
  66 #    define UNW_DEBUG_ON(n)     unw_debug_level >= n
  67      /* Do not code a printk level, not all debug lines end in newline */
  68 #    define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  69 #  endif /* CONFIG_KDB */
  70 #  define inline
  71 #else /* !UNW_DEBUG */
  72 #  define UNW_DEBUG_ON(n)  0
  73 #  define UNW_DPRINT(n, ...)
  74 #endif /* UNW_DEBUG */
  75
  76 #if UNW_STATS
  77 # define STAT(x...)     x
  78 #else
  79 # define STAT(x...)
  80 #endif
  81
  82 #define alloc_reg_state()       kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
  83 #define free_reg_state(usr)     kfree(usr)
  84 #define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  85 #define free_labeled_state(usr) kfree(usr)
  86
  87 typedef unsigned long unw_word;
  88 typedef unsigned char unw_hash_index_t;
  89
  90 static struct {
  91         spinlock_t lock;                        /* spinlock for unwind data */
  92
  93         /* list of unwind tables (one per load-module) */
  94         struct unw_table *tables;
  95
  96         /* table of registers that prologues can save (and order in which they're saved): */
  97         const unsigned char save_order[8];
  98
  99         /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
 100         unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
 101
 102         unsigned short lru_head;                /* index of lead-recently used script */
 103         unsigned short lru_tail;                /* index of most-recently used script */
 104
 105         /* index into unw_frame_info for preserved register i */
 106         unsigned short preg_index[UNW_NUM_REGS];
 107
 108         short pt_regs_offsets[32];
 109
 110         /* unwind table for the kernel: */
 111         struct unw_table kernel_table;
 112
 113         /* unwind table describing the gate page (kernel code that is mapped into user space): */
 114         size_t gate_table_size;
 115         unsigned long *gate_table;
 116
 117         /* hash table that maps instruction pointer to script index: */
 118         unsigned short hash[UNW_HASH_SIZE];
 119
 120         /* script cache: */
 121         struct unw_script cache[UNW_CACHE_SIZE];
 122
 123 # ifdef UNW_DEBUG
 124         const char *preg_name[UNW_NUM_REGS];
 125 # endif
 126 # if UNW_STATS
 127         struct {
 128                 struct {
 129                         int lookups;
 130                         int hinted_hits;
 131                         int normal_hits;
 132                         int collision_chain_traversals;
 133                 } cache;
 134                 struct {
 135                         unsigned long build_time;
 136                         unsigned long run_time;
 137                         unsigned long parse_time;
 138                         int builds;
 139                         int news;
 140                         int collisions;
 141                         int runs;
 142                 } script;
 143                 struct {
 144                         unsigned long init_time;
 145                         unsigned long unwind_time;
 146                         int inits;
 147                         int unwinds;
 148                 } api;
 149         } stat;
 150 # endif
 151 } unw = {
 152         .tables = &unw.kernel_table,
 153         .lock = SPIN_LOCK_UNLOCKED,
 154         .save_order = {
 155                 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 156                 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 157         },
 158         .preg_index = {
 159                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_GR */
 160                 offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_MEM */
 161                 offsetof(struct unw_frame_info, bsp_loc)/8,
 162                 offsetof(struct unw_frame_info, bspstore_loc)/8,
 163                 offsetof(struct unw_frame_info, pfs_loc)/8,
 164                 offsetof(struct unw_frame_info, rnat_loc)/8,
 165                 offsetof(struct unw_frame_info, psp)/8,
 166                 offsetof(struct unw_frame_info, rp_loc)/8,
 167                 offsetof(struct unw_frame_info, r4)/8,
 168                 offsetof(struct unw_frame_info, r5)/8,
 169                 offsetof(struct unw_frame_info, r6)/8,
 170                 offsetof(struct unw_frame_info, r7)/8,
 171                 offsetof(struct unw_frame_info, unat_loc)/8,
 172                 offsetof(struct unw_frame_info, pr_loc)/8,
 173                 offsetof(struct unw_frame_info, lc_loc)/8,
 174                 offsetof(struct unw_frame_info, fpsr_loc)/8,
 175                 offsetof(struct unw_frame_info, b1_loc)/8,
 176                 offsetof(struct unw_frame_info, b2_loc)/8,
 177                 offsetof(struct unw_frame_info, b3_loc)/8,
 178                 offsetof(struct unw_frame_info, b4_loc)/8,
 179                 offsetof(struct unw_frame_info, b5_loc)/8,
 180                 offsetof(struct unw_frame_info, f2_loc)/8,
 181                 offsetof(struct unw_frame_info, f3_loc)/8,
 182                 offsetof(struct unw_frame_info, f4_loc)/8,
 183                 offsetof(struct unw_frame_info, f5_loc)/8,
 184                 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
 185                 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
 186                 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
 187                 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
 188                 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
 189                 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
 190                 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
 191                 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
 192                 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
 193                 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
 194                 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
 195                 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
 196                 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
 197                 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
 198                 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
 199                 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
 200         },
 201         .pt_regs_offsets = {
 202                 [0] = -1,
 203                 offsetof(struct pt_regs,  r1),
 204                 offsetof(struct pt_regs,  r2),
 205                 offsetof(struct pt_regs,  r3),
 206                 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
 207                 offsetof(struct pt_regs,  r8),
 208                 offsetof(struct pt_regs,  r9),
 209                 offsetof(struct pt_regs, r10),
 210                 offsetof(struct pt_regs, r11),
 211                 offsetof(struct pt_regs, r12),
 212                 offsetof(struct pt_regs, r13),
 213                 offsetof(struct pt_regs, r14),
 214                 offsetof(struct pt_regs, r15),
 215                 offsetof(struct pt_regs, r16),
 216                 offsetof(struct pt_regs, r17),
 217                 offsetof(struct pt_regs, r18),
 218                 offsetof(struct pt_regs, r19),
 219                 offsetof(struct pt_regs, r20),
 220                 offsetof(struct pt_regs, r21),
 221                 offsetof(struct pt_regs, r22),
 222                 offsetof(struct pt_regs, r23),
 223                 offsetof(struct pt_regs, r24),
 224                 offsetof(struct pt_regs, r25),
 225                 offsetof(struct pt_regs, r26),
 226                 offsetof(struct pt_regs, r27),
 227                 offsetof(struct pt_regs, r28),
 228                 offsetof(struct pt_regs, r29),
 229                 offsetof(struct pt_regs, r30),
 230                 offsetof(struct pt_regs, r31),
 231         },
 232         .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 233 #ifdef UNW_DEBUG
 234         .preg_name = {
 235                 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 236                 "r4", "r5", "r6", "r7",
 237                 "ar.unat", "pr", "ar.lc", "ar.fpsr",
 238                 "b1", "b2", "b3", "b4", "b5",
 239                 "f2", "f3", "f4", "f5",
 240                 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 241                 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 242         }
 243 #endif
 244 };
 245
 246 /* Unwind accessors.  */
 247
 248 /*
 249  * Returns offset of rREG in struct pt_regs.
 250  */
 251 static inline unsigned long
 252 pt_regs_off (unsigned long reg)
 253 {
 254         short off = -1;
 255
 256         if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
 257                 off = unw.pt_regs_offsets[reg];
 258
 259         if (off < 0) {
 260                 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
 261                 off = 0;
 262         }
 263         return (unsigned long) off;
 264 }
 265
 266 static inline struct pt_regs *
 267 get_scratch_regs (struct unw_frame_info *info)
 268 {
 269         if (!info->pt) {
 270                 /* This should not happen with valid unwind info.  */
 271                 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
 272                 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 273                         info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 274                 else
 275                         info->pt = info->sp - 16;
 276         }
 277         UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
 278         return (struct pt_regs *) info->pt;
 279 }
 280
 281 int
 282 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 283 {
 284         unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 285         struct unw_ireg *ireg;
 286         struct pt_regs *pt;
 287
 288         if ((unsigned) regnum - 1 >= 127) {
 289                 if (regnum == 0 && !write) {
 290                         *val = 0;       /* read r0 always returns 0 */
 291                         *nat = 0;
 292                         return 0;
 293                 }
 294                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 295                            __FUNCTION__, regnum);
 296                 return -1;
 297         }
 298
 299         if (regnum < 32) {
 300                 if (regnum >= 4 && regnum <= 7) {
 301                         /* access a preserved register */
 302                         ireg = &info->r4 + (regnum - 4);
 303                         addr = ireg->loc;
 304                         if (addr) {
 305                                 nat_addr = addr + ireg->nat.off;
 306                                 switch (ireg->nat.type) {
 307                                       case UNW_NAT_VAL:
 308                                         /* simulate getf.sig/setf.sig */
 309                                         if (write) {
 310                                                 if (*nat) {
 311                                                         /* write NaTVal and be done with it */
 312                                                         addr[0] = 0;
 313                                                         addr[1] = 0x1fffe;
 314                                                         return 0;
 315                                                 }
 316                                                 addr[1] = 0x1003e;
 317                                         } else {
 318                                                 if (addr[0] == 0 && addr[1] == 0x1ffe) {
 319                                                         /* return NaT and be done with it */
 320                                                         *val = 0;
 321                                                         *nat = 1;
 322                                                         return 0;
 323                                                 }
 324                                         }
 325                                         /* fall through */
 326                                       case UNW_NAT_NONE:
 327                                         dummy_nat = 0;
 328                                         nat_addr = &dummy_nat;
 329                                         break;
 330
 331                                       case UNW_NAT_MEMSTK:
 332                                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 333                                         break;
 334
 335                                       case UNW_NAT_REGSTK:
 336                                         nat_addr = ia64_rse_rnat_addr(addr);
 337                                         if ((unsigned long) addr < info->regstk.limit
 338                                             || (unsigned long) addr >= info->regstk.top)
 339                                         {
 340                                                 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 341                                                         "[0x%lx-0x%lx)\n",
 342                                                         __FUNCTION__, (void *) addr,
 343                                                         info->regstk.limit,
 344                                                         info->regstk.top);
 345                                                 return -1;
 346                                         }
 347                                         if ((unsigned long) nat_addr >= info->regstk.top)
 348                                                 nat_addr = &info->sw->ar_rnat;
 349                                         nat_mask = (1UL << ia64_rse_slot_num(addr));
 350                                         break;
 351                                 }
 352                         } else {
 353                                 addr = &info->sw->r4 + (regnum - 4);
 354                                 nat_addr = &info->sw->ar_unat;
 355                                 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 356                         }
 357                 } else {
 358                         /* access a scratch register */
 359                         pt = get_scratch_regs(info);
 360                         addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 361                         if (info->pri_unat_loc)
 362                                 nat_addr = info->pri_unat_loc;
 363                         else
 364                                 nat_addr = &info->sw->ar_unat;
 365                         nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 366                 }
 367         } else {
 368                 /* access a stacked register */
 369                 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 370                 nat_addr = ia64_rse_rnat_addr(addr);
 371                 if ((unsigned long) addr < info->regstk.limit
 372                     || (unsigned long) addr >= info->regstk.top)
 373                 {
 374                         UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 375                                    "of rbs\n",  __FUNCTION__);
 376                         return -1;
 377                 }
 378                 if ((unsigned long) nat_addr >= info->regstk.top)
 379                         nat_addr = &info->sw->ar_rnat;
 380                 nat_mask = (1UL << ia64_rse_slot_num(addr));
 381         }
 382
 383         if (write) {
 384                 *addr = *val;
 385                 if (*nat)
 386                         *nat_addr |= nat_mask;
 387                 else
 388                         *nat_addr &= ~nat_mask;
 389         } else {
 390                 if ((*nat_addr & nat_mask) == 0) {
 391                         *val = *addr;
 392                         *nat = 0;
 393                 } else {
 394                         *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 395                         *nat = 1;
 396                 }
 397         }
 398         return 0;
 399 }
 400
 401 int
 402 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 403 {
 404         unsigned long *addr;
 405         struct pt_regs *pt;
 406
 407         switch (regnum) {
 408                 /* scratch: */
 409               case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
 410               case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
 411               case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
 412
 413                 /* preserved: */
 414               case 1: case 2: case 3: case 4: case 5:
 415                 addr = *(&info->b1_loc + (regnum - 1));
 416                 if (!addr)
 417                         addr = &info->sw->b1 + (regnum - 1);
 418                 break;
 419
 420               default:
 421                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 422                            __FUNCTION__, regnum);
 423                 return -1;
 424         }
 425         if (write)
 426                 *addr = *val;
 427         else
 428                 *val = *addr;
 429         return 0;
 430 }
 431
 432 int
 433 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 434 {
 435         struct ia64_fpreg *addr = 0;
 436         struct pt_regs *pt;
 437
 438         if ((unsigned) (regnum - 2) >= 126) {
 439                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 440                            __FUNCTION__, regnum);
 441                 return -1;
 442         }
 443
 444         if (regnum <= 5) {
 445                 addr = *(&info->f2_loc + (regnum - 2));
 446                 if (!addr)
 447                         addr = &info->sw->f2 + (regnum - 2);
 448         } else if (regnum <= 15) {
 449                 if (regnum <= 11) {
 450                         pt = get_scratch_regs(info);
 451                         addr = &pt->f6  + (regnum - 6);
 452                 }
 453                 else
 454                         addr = &info->sw->f12 + (regnum - 12);
 455         } else if (regnum <= 31) {
 456                 addr = info->fr_loc[regnum - 16];
 457                 if (!addr)
 458                         addr = &info->sw->f16 + (regnum - 16);
 459         } else {
 460                 struct task_struct *t = info->task;
 461
 462                 if (write)
 463                         ia64_sync_fph(t);
 464                 else
 465                         ia64_flush_fph(t);
 466                 addr = t->thread.fph + (regnum - 32);
 467         }
 468
 469         if (write)
 470                 *addr = *val;
 471         else
 472                 *val = *addr;
 473         return 0;
 474 }
 475
 476 int
 477 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 478 {
 479         unsigned long *addr;
 480         struct pt_regs *pt;
 481
 482         switch (regnum) {
 483               case UNW_AR_BSP:
 484                 addr = info->bsp_loc;
 485                 if (!addr)
 486                         addr = &info->sw->ar_bspstore;
 487                 break;
 488
 489               case UNW_AR_BSPSTORE:
 490                 addr = info->bspstore_loc;
 491                 if (!addr)
 492                         addr = &info->sw->ar_bspstore;
 493                 break;
 494
 495               case UNW_AR_PFS:
 496                 addr = info->pfs_loc;
 497                 if (!addr)
 498                         addr = &info->sw->ar_pfs;
 499                 break;
 500
 501               case UNW_AR_RNAT:
 502                 addr = info->rnat_loc;
 503                 if (!addr)
 504                         addr = &info->sw->ar_rnat;
 505                 break;
 506
 507               case UNW_AR_UNAT:
 508                 addr = info->unat_loc;
 509                 if (!addr)
 510                         addr = &info->sw->ar_unat;
 511                 break;
 512
 513               case UNW_AR_LC:
 514                 addr = info->lc_loc;
 515                 if (!addr)
 516                         addr = &info->sw->ar_lc;
 517                 break;
 518
 519               case UNW_AR_EC:
 520                 if (!info->cfm_loc)
 521                         return -1;
 522                 if (write)
 523                         *info->cfm_loc =
 524                                 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 525                 else
 526                         *val = (*info->cfm_loc >> 52) & 0x3f;
 527                 return 0;
 528
 529               case UNW_AR_FPSR:
 530                 addr = info->fpsr_loc;
 531                 if (!addr)
 532                         addr = &info->sw->ar_fpsr;
 533                 break;
 534
 535               case UNW_AR_RSC:
 536                 pt = get_scratch_regs(info);
 537                 addr = &pt->ar_rsc;
 538                 break;
 539
 540               case UNW_AR_CCV:
 541                 pt = get_scratch_regs(info);
 542                 addr = &pt->ar_ccv;
 543                 break;
 544
 545               case UNW_AR_CSD:
 546                 pt = get_scratch_regs(info);
 547                 addr = &pt->ar_csd;
 548                 break;
 549
 550               case UNW_AR_SSD:
 551                 pt = get_scratch_regs(info);
 552                 addr = &pt->ar_ssd;
 553                 break;
 554
 555               default:
 556                 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 557                            __FUNCTION__, regnum);
 558                 return -1;
 559         }
 560
 561         if (write)
 562                 *addr = *val;
 563         else
 564                 *val = *addr;
 565         return 0;
 566 }
 567
 568 int
 569 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 570 {
 571         unsigned long *addr;
 572
 573         addr = info->pr_loc;
 574         if (!addr)
 575                 addr = &info->sw->pr;
 576
 577         if (write)
 578                 *addr = *val;
 579         else
 580                 *val = *addr;
 581         return 0;
 582 }
 583
 584 \f
 585 /* Routines to manipulate the state stack.  */
 586
 587 static inline void
 588 push (struct unw_state_record *sr)
 589 {
 590         struct unw_reg_state *rs;
 591
 592         rs = alloc_reg_state();
 593         if (!rs) {
 594                 printk(KERN_ERR "unwind: cannot stack reg state!\n");
 595                 return;
 596         }
 597         memcpy(rs, &sr->curr, sizeof(*rs));
 598         sr->curr.next = rs;
 599 }
 600
 601 static void
 602 pop (struct unw_state_record *sr)
 603 {
 604         struct unw_reg_state *rs = sr->curr.next;
 605
 606         if (!rs) {
 607                 printk(KERN_ERR "unwind: stack underflow!\n");
 608                 return;
 609         }
 610         memcpy(&sr->curr, rs, sizeof(*rs));
 611         free_reg_state(rs);
 612 }
 613
 614 /* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 615 static struct unw_reg_state *
 616 dup_state_stack (struct unw_reg_state *rs)
 617 {
 618         struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 619
 620         while (rs) {
 621                 copy = alloc_reg_state();
 622                 if (!copy) {
 623                         printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 624                         return NULL;
 625                 }
 626                 memcpy(copy, rs, sizeof(*copy));
 627                 if (first)
 628                         prev->next = copy;
 629                 else
 630                         first = copy;
 631                 rs = rs->next;
 632                 prev = copy;
 633         }
 634         return first;
 635 }
 636
 637 /* Free all stacked register states (but not RS itself).  */
 638 static void
 639 free_state_stack (struct unw_reg_state *rs)
 640 {
 641         struct unw_reg_state *p, *next;
 642
 643         for (p = rs->next; p != NULL; p = next) {
 644                 next = p->next;
 645                 free_reg_state(p);
 646         }
 647         rs->next = NULL;
 648 }
 649 \f
 650 /* Unwind decoder routines */
 651
 652 static enum unw_register_index __attribute__((const))
 653 decode_abreg (unsigned char abreg, int memory)
 654 {
 655         switch (abreg) {
 656               case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 657               case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 658               case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 659               case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 660               case 0x60: return UNW_REG_PR;
 661               case 0x61: return UNW_REG_PSP;
 662               case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 663               case 0x63: return UNW_REG_RP;
 664               case 0x64: return UNW_REG_BSP;
 665               case 0x65: return UNW_REG_BSPSTORE;
 666               case 0x66: return UNW_REG_RNAT;
 667               case 0x67: return UNW_REG_UNAT;
 668               case 0x68: return UNW_REG_FPSR;
 669               case 0x69: return UNW_REG_PFS;
 670               case 0x6a: return UNW_REG_LC;
 671               default:
 672                 break;
 673         }
 674         UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
 675         return UNW_REG_LC;
 676 }
 677
 678 static void
 679 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 680 {
 681         reg->val = val;
 682         reg->where = where;
 683         if (reg->when == UNW_WHEN_NEVER)
 684                 reg->when = when;
 685 }
 686
 687 static void
 688 alloc_spill_area (unsigned long *offp, unsigned long regsize,
 689                   struct unw_reg_info *lo, struct unw_reg_info *hi)
 690 {
 691         struct unw_reg_info *reg;
 692
 693         for (reg = hi; reg >= lo; --reg) {
 694                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 695                         reg->where = UNW_WHERE_PSPREL;
 696                         *offp -= regsize;
 697                         reg->val = *offp;
 698                 }
 699         }
 700 }
 701
 702 static inline void
 703 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 704 {
 705         struct unw_reg_info *reg;
 706
 707         for (reg = *regp; reg <= lim; ++reg) {
 708                 if (reg->where == UNW_WHERE_SPILL_HOME) {
 709                         reg->when = t;
 710                         *regp = reg + 1;
 711                         return;
 712                 }
 713         }
 714         UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __FUNCTION__);
 715 }
 716
 717 static inline void
 718 finish_prologue (struct unw_state_record *sr)
 719 {
 720         struct unw_reg_info *reg;
 721         unsigned long off;
 722         int i;
 723
 724         /*
 725          * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 726          * for Using Unwind Descriptors", rule 3):
 727          */
 728         for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 729                 reg = sr->curr.reg + unw.save_order[i];
 730                 if (reg->where == UNW_WHERE_GR_SAVE) {
 731                         reg->where = UNW_WHERE_GR;
 732                         reg->val = sr->gr_save_loc++;
 733                 }
 734         }
 735
 736         /*
 737          * Next, compute when the fp, general, and branch registers get
 738          * saved.  This must come before alloc_spill_area() because
 739          * we need to know which registers are spilled to their home
 740          * locations.
 741          */
 742         if (sr->imask) {
 743                 unsigned char kind, mask = 0, *cp = sr->imask;
 744                 int t;
 745                 static const unsigned char limit[3] = {
 746                         UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 747                 };
 748                 struct unw_reg_info *(regs[3]);
 749
 750                 regs[0] = sr->curr.reg + UNW_REG_F2;
 751                 regs[1] = sr->curr.reg + UNW_REG_R4;
 752                 regs[2] = sr->curr.reg + UNW_REG_B1;
 753
 754                 for (t = 0; t < sr->region_len; ++t) {
 755                         if ((t & 3) == 0)
 756                                 mask = *cp++;
 757                         kind = (mask >> 2*(3-(t & 3))) & 3;
 758                         if (kind > 0)
 759                                 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 760                                                 sr->region_start + t);
 761                 }
 762         }
 763         /*
 764          * Next, lay out the memory stack spill area:
 765          */
 766         if (sr->any_spills) {
 767                 off = sr->spill_offset;
 768                 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 769                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 770                 alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 771         }
 772 }
 773
 774 /*
 775  * Region header descriptors.
 776  */
 777
 778 static void
 779 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 780                struct unw_state_record *sr)
 781 {
 782         int i, region_start;
 783
 784         if (!(sr->in_body || sr->first_region))
 785                 finish_prologue(sr);
 786         sr->first_region = 0;
 787
 788         /* check if we're done: */
 789         if (sr->when_target < sr->region_start + sr->region_len) {
 790                 sr->done = 1;
 791                 return;
 792         }
 793
 794         region_start = sr->region_start + sr->region_len;
 795
 796         for (i = 0; i < sr->epilogue_count; ++i)
 797                 pop(sr);
 798         sr->epilogue_count = 0;
 799         sr->epilogue_start = UNW_WHEN_NEVER;
 800
 801         sr->region_start = region_start;
 802         sr->region_len = rlen;
 803         sr->in_body = body;
 804
 805         if (!body) {
 806                 push(sr);
 807
 808                 for (i = 0; i < 4; ++i) {
 809                         if (mask & 0x8)
 810                                 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 811                                         sr->region_start + sr->region_len - 1, grsave++);
 812                         mask <<= 1;
 813                 }
 814                 sr->gr_save_loc = grsave;
 815                 sr->any_spills = 0;
 816                 sr->imask = 0;
 817                 sr->spill_offset = 0x10;        /* default to psp+16 */
 818         }
 819 }
 820
 821 /*
 822  * Prologue descriptors.
 823  */
 824
 825 static inline void
 826 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 827 {
 828         if (abi == 3 && context == 'i') {
 829                 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 830                 UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __FUNCTION__);
 831         }
 832         else
 833                 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 834                                 __FUNCTION__, abi, context);
 835 }
 836
 837 static inline void
 838 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 839 {
 840         int i;
 841
 842         for (i = 0; i < 5; ++i) {
 843                 if (brmask & 1)
 844                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 845                                 sr->region_start + sr->region_len - 1, gr++);
 846                 brmask >>= 1;
 847         }
 848 }
 849
 850 static inline void
 851 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 852 {
 853         int i;
 854
 855         for (i = 0; i < 5; ++i) {
 856                 if (brmask & 1) {
 857                         set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 858                                 sr->region_start + sr->region_len - 1, 0);
 859                         sr->any_spills = 1;
 860                 }
 861                 brmask >>= 1;
 862         }
 863 }
 864
 865 static inline void
 866 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 867 {
 868         int i;
 869
 870         for (i = 0; i < 4; ++i) {
 871                 if ((grmask & 1) != 0) {
 872                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 873                                 sr->region_start + sr->region_len - 1, 0);
 874                         sr->any_spills = 1;
 875                 }
 876                 grmask >>= 1;
 877         }
 878         for (i = 0; i < 20; ++i) {
 879                 if ((frmask & 1) != 0) {
 880                         int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 881                         set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 882                                 sr->region_start + sr->region_len - 1, 0);
 883                         sr->any_spills = 1;
 884                 }
 885                 frmask >>= 1;
 886         }
 887 }
 888
 889 static inline void
 890 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 891 {
 892         int i;
 893
 894         for (i = 0; i < 4; ++i) {
 895                 if ((frmask & 1) != 0) {
 896                         set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 897                                 sr->region_start + sr->region_len - 1, 0);
 898                         sr->any_spills = 1;
 899                 }
 900                 frmask >>= 1;
 901         }
 902 }
 903
 904 static inline void
 905 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 906 {
 907         int i;
 908
 909         for (i = 0; i < 4; ++i) {
 910                 if ((grmask & 1) != 0)
 911                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 912                                 sr->region_start + sr->region_len - 1, gr++);
 913                 grmask >>= 1;
 914         }
 915 }
 916
 917 static inline void
 918 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 919 {
 920         int i;
 921
 922         for (i = 0; i < 4; ++i) {
 923                 if ((grmask & 1) != 0) {
 924                         set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 925                                 sr->region_start + sr->region_len - 1, 0);
 926                         sr->any_spills = 1;
 927                 }
 928                 grmask >>= 1;
 929         }
 930 }
 931
 932 static inline void
 933 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 934 {
 935         set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 936                 sr->region_start + MIN((int)t, sr->region_len - 1), 16*size);
 937 }
 938
 939 static inline void
 940 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 941 {
 942         sr->curr.reg[UNW_REG_PSP].when = sr->region_start + MIN((int)t, sr->region_len - 1);
 943 }
 944
 945 static inline void
 946 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 947 {
 948         set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 949 }
 950
 951 static inline void
 952 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 953 {
 954         set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 955                 0x10 - 4*pspoff);
 956 }
 957
 958 static inline void
 959 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 960 {
 961         set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 962                 4*spoff);
 963 }
 964
 965 static inline void
 966 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 967 {
 968         sr->return_link_reg = dst;
 969 }
 970
 971 static inline void
 972 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
 973 {
 974         struct unw_reg_info *reg = sr->curr.reg + regnum;
 975
 976         if (reg->where == UNW_WHERE_NONE)
 977                 reg->where = UNW_WHERE_GR_SAVE;
 978         reg->when = sr->region_start + MIN((int)t, sr->region_len - 1);
 979 }
 980
 981 static inline void
 982 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
 983 {
 984         sr->spill_offset = 0x10 - 4*pspoff;
 985 }
 986
 987 static inline unsigned char *
 988 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
 989 {
 990         sr->imask = imaskp;
 991         return imaskp + (2*sr->region_len + 7)/8;
 992 }
 993
 994 /*
 995  * Body descriptors.
 996  */
 997 static inline void
 998 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
 999 {
1000         sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1001         sr->epilogue_count = ecount + 1;
1002 }
1003
1004 static inline void
1005 desc_copy_state (unw_word label, struct unw_state_record *sr)
1006 {
1007         struct unw_labeled_state *ls;
1008
1009         for (ls = sr->labeled_states; ls; ls = ls->next) {
1010                 if (ls->label == label) {
1011                         free_state_stack(&sr->curr);
1012                         memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1013                         sr->curr.next = dup_state_stack(ls->saved_state.next);
1014                         return;
1015                 }
1016         }
1017         printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1018 }
1019
1020 static inline void
1021 desc_label_state (unw_word label, struct unw_state_record *sr)
1022 {
1023         struct unw_labeled_state *ls;
1024
1025         ls = alloc_labeled_state();
1026         if (!ls) {
1027                 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1028                 return;
1029         }
1030         ls->label = label;
1031         memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1032         ls->saved_state.next = dup_state_stack(sr->curr.next);
1033
1034         /* insert into list of labeled states: */
1035         ls->next = sr->labeled_states;
1036         sr->labeled_states = ls;
1037 }
1038
1039 /*
1040  * General descriptors.
1041  */
1042
1043 static inline int
1044 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1045 {
1046         if (sr->when_target <= sr->region_start + MIN((int)t, sr->region_len - 1))
1047                 return 0;
1048         if (qp > 0) {
1049                 if ((sr->pr_val & (1UL << qp)) == 0)
1050                         return 0;
1051                 sr->pr_mask |= (1UL << qp);
1052         }
1053         return 1;
1054 }
1055
1056 static inline void
1057 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1058 {
1059         struct unw_reg_info *r;
1060
1061         if (!desc_is_active(qp, t, sr))
1062                 return;
1063
1064         r = sr->curr.reg + decode_abreg(abreg, 0);
1065         r->where = UNW_WHERE_NONE;
1066         r->when = UNW_WHEN_NEVER;
1067         r->val = 0;
1068 }
1069
1070 static inline void
1071 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1072                      unsigned char ytreg, struct unw_state_record *sr)
1073 {
1074         enum unw_where where = UNW_WHERE_GR;
1075         struct unw_reg_info *r;
1076
1077         if (!desc_is_active(qp, t, sr))
1078                 return;
1079
1080         if (x)
1081                 where = UNW_WHERE_BR;
1082         else if (ytreg & 0x80)
1083                 where = UNW_WHERE_FR;
1084
1085         r = sr->curr.reg + decode_abreg(abreg, 0);
1086         r->where = where;
1087         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1088         r->val = (ytreg & 0x7f);
1089 }
1090
1091 static inline void
1092 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1093                      struct unw_state_record *sr)
1094 {
1095         struct unw_reg_info *r;
1096
1097         if (!desc_is_active(qp, t, sr))
1098                 return;
1099
1100         r = sr->curr.reg + decode_abreg(abreg, 1);
1101         r->where = UNW_WHERE_PSPREL;
1102         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1103         r->val = 0x10 - 4*pspoff;
1104 }
1105
1106 static inline void
1107 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1108                        struct unw_state_record *sr)
1109 {
1110         struct unw_reg_info *r;
1111
1112         if (!desc_is_active(qp, t, sr))
1113                 return;
1114
1115         r = sr->curr.reg + decode_abreg(abreg, 1);
1116         r->where = UNW_WHERE_SPREL;
1117         r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1118         r->val = 4*spoff;
1119 }
1120
1121 #define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1122                                                        code);
1123
1124 /*
1125  * region headers:
1126  */
1127 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1128 #define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1129 /*
1130  * prologue descriptors:
1131  */
1132 #define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1133 #define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1134 #define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1135 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1136 #define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1137 #define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1138 #define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1139 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1140 #define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1141 #define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1142 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1143 #define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1144 #define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1145 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1146 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1147 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1148 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1149 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1150 #define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1151 #define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1152 #define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1153 /*
1154  * body descriptors:
1155  */
1156 #define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1157 #define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1158 #define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1159 /*
1160  * general unwind descriptors:
1161  */
1162 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1163 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1164 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1165 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1166 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1167 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1168 #define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1169 #define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1170
1171 #include "unwind_decoder.c"
1172
1173 \f
1174 /* Unwind scripts. */
1175
1176 static inline unw_hash_index_t
1177 hash (unsigned long ip)
1178 {
1179 #       define magic    0x9e3779b97f4a7c16      /* based on (sqrt(5)/2-1)*2^64 */
1180
1181         return (ip >> 4)*magic >> (64 - UNW_LOG_HASH_SIZE);
1182 }
1183
1184 static inline long
1185 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1186 {
1187         read_lock(&script->lock);
1188         if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1189                 /* keep the read lock... */
1190                 return 1;
1191         read_unlock(&script->lock);
1192         return 0;
1193 }
1194
1195 static inline struct unw_script *
1196 script_lookup (struct unw_frame_info *info)
1197 {
1198         struct unw_script *script = unw.cache + info->hint;
1199         unsigned short index;
1200         unsigned long ip, pr;
1201
1202         if (UNW_DEBUG_ON(0))
1203                 return 0;       /* Always regenerate scripts in debug mode */
1204
1205         STAT(++unw.stat.cache.lookups);
1206
1207         ip = info->ip;
1208         pr = info->pr;
1209
1210         if (cache_match(script, ip, pr)) {
1211                 STAT(++unw.stat.cache.hinted_hits);
1212                 return script;
1213         }
1214
1215         index = unw.hash[hash(ip)];
1216         if (index >= UNW_CACHE_SIZE)
1217                 return 0;
1218
1219         script = unw.cache + index;
1220         while (1) {
1221                 if (cache_match(script, ip, pr)) {
1222                         /* update hint; no locking required as single-word writes are atomic */
1223                         STAT(++unw.stat.cache.normal_hits);
1224                         unw.cache[info->prev_script].hint = script - unw.cache;
1225                         return script;
1226                 }
1227                 if (script->coll_chain >= UNW_HASH_SIZE)
1228                         return 0;
1229                 script = unw.cache + script->coll_chain;
1230                 STAT(++unw.stat.cache.collision_chain_traversals);
1231         }
1232 }
1233
1234 /*
1235  * On returning, a write lock for the SCRIPT is still being held.
1236  */
1237 static inline struct unw_script *
1238 script_new (unsigned long ip)
1239 {
1240         struct unw_script *script, *prev, *tmp;
1241         unw_hash_index_t index;
1242         unsigned long flags;
1243         unsigned short head;
1244
1245         STAT(++unw.stat.script.news);
1246
1247         /*
1248          * Can't (easily) use cmpxchg() here because of ABA problem
1249          * that is intrinsic in cmpxchg()...
1250          */
1251         spin_lock_irqsave(&unw.lock, flags);
1252         {
1253                 head = unw.lru_head;
1254                 script = unw.cache + head;
1255                 unw.lru_head = script->lru_chain;
1256         }
1257         spin_unlock(&unw.lock);
1258
1259         /*
1260          * We'd deadlock here if we interrupted a thread that is holding a read lock on
1261          * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1262          * alternative would be to disable interrupts whenever we hold a read-lock, but
1263          * that seems silly.
1264          */
1265         if (!write_trylock(&script->lock))
1266                 return NULL;
1267
1268         spin_lock(&unw.lock);
1269         {
1270                 /* re-insert script at the tail of the LRU chain: */
1271                 unw.cache[unw.lru_tail].lru_chain = head;
1272                 unw.lru_tail = head;
1273
1274                 /* remove the old script from the hash table (if it's there): */
1275                 if (script->ip) {
1276                         index = hash(script->ip);
1277                         tmp = unw.cache + unw.hash[index];
1278                         prev = 0;
1279                         while (1) {
1280                                 if (tmp == script) {
1281                                         if (prev)
1282                                                 prev->coll_chain = tmp->coll_chain;
1283                                         else
1284                                                 unw.hash[index] = tmp->coll_chain;
1285                                         break;
1286                                 } else
1287                                         prev = tmp;
1288                                 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1289                                 /* old script wasn't in the hash-table */
1290                                         break;
1291                                 tmp = unw.cache + tmp->coll_chain;
1292                         }
1293                 }
1294
1295                 /* enter new script in the hash table */
1296                 index = hash(ip);
1297                 script->coll_chain = unw.hash[index];
1298                 unw.hash[index] = script - unw.cache;
1299
1300                 script->ip = ip;        /* set new IP while we're holding the locks */
1301
1302                 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1303         }
1304         spin_unlock_irqrestore(&unw.lock, flags);
1305
1306         script->flags = 0;
1307         script->hint = 0;
1308         script->count = 0;
1309         return script;
1310 }
1311
1312 static void
1313 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1314 {
1315         script->pr_mask = sr->pr_mask;
1316         script->pr_val = sr->pr_val;
1317         /*
1318          * We could down-grade our write-lock on script->lock here but
1319          * the rwlock API doesn't offer atomic lock downgrading, so
1320          * we'll just keep the write-lock and release it later when
1321          * we're done using the script.
1322          */
1323 }
1324
1325 static inline void
1326 script_emit (struct unw_script *script, struct unw_insn insn)
1327 {
1328         if (script->count >= UNW_MAX_SCRIPT_LEN) {
1329                 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1330                         __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1331                 return;
1332         }
1333         script->insn[script->count++] = insn;
1334 }
1335
1336 static inline void
1337 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1338 {
1339         struct unw_reg_info *r = sr->curr.reg + i;
1340         enum unw_insn_opcode opc;
1341         struct unw_insn insn;
1342         unsigned long val = 0;
1343
1344         switch (r->where) {
1345               case UNW_WHERE_GR:
1346                 if (r->val >= 32) {
1347                         /* register got spilled to a stacked register */
1348                         opc = UNW_INSN_SETNAT_TYPE;
1349                         val = UNW_NAT_REGSTK;
1350                 } else
1351                         /* register got spilled to a scratch register */
1352                         opc = UNW_INSN_SETNAT_MEMSTK;
1353                 break;
1354
1355               case UNW_WHERE_FR:
1356                 opc = UNW_INSN_SETNAT_TYPE;
1357                 val = UNW_NAT_VAL;
1358                 break;
1359
1360               case UNW_WHERE_BR:
1361                 opc = UNW_INSN_SETNAT_TYPE;
1362                 val = UNW_NAT_NONE;
1363                 break;
1364
1365               case UNW_WHERE_PSPREL:
1366               case UNW_WHERE_SPREL:
1367                 opc = UNW_INSN_SETNAT_MEMSTK;
1368                 break;
1369
1370               default:
1371                 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1372                            __FUNCTION__, r->where);
1373                 return;
1374         }
1375         insn.opc = opc;
1376         insn.dst = unw.preg_index[i];
1377         insn.val = val;
1378         script_emit(script, insn);
1379 }
1380
1381 static void
1382 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1383 {
1384         struct unw_reg_info *r = sr->curr.reg + i;
1385         enum unw_insn_opcode opc;
1386         unsigned long val, rval;
1387         struct unw_insn insn;
1388         long need_nat_info;
1389
1390         if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1391                 return;
1392
1393         opc = UNW_INSN_MOVE;
1394         val = rval = r->val;
1395         need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1396
1397         switch (r->where) {
1398               case UNW_WHERE_GR:
1399                 if (rval >= 32) {
1400                         opc = UNW_INSN_MOVE_STACKED;
1401                         val = rval - 32;
1402                 } else if (rval >= 4 && rval <= 7) {
1403                         if (need_nat_info) {
1404                                 opc = UNW_INSN_MOVE2;
1405                                 need_nat_info = 0;
1406                         }
1407                         val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1408                 } else {
1409                         /* register got spilled to a scratch register */
1410                         opc = UNW_INSN_MOVE_SCRATCH;
1411                         val = pt_regs_off(rval);
1412                 }
1413                 break;
1414
1415               case UNW_WHERE_FR:
1416                 if (rval <= 5)
1417                         val = unw.preg_index[UNW_REG_F2  + (rval -  1)];
1418                 else if (rval >= 16 && rval <= 31)
1419                         val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1420                 else {
1421                         opc = UNW_INSN_MOVE_SCRATCH;
1422                         if (rval <= 11)
1423                                 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1424                         else
1425                                 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1426                                            __FUNCTION__, rval);
1427                 }
1428                 break;
1429
1430               case UNW_WHERE_BR:
1431                 if (rval >= 1 && rval <= 5)
1432                         val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1433                 else {
1434                         opc = UNW_INSN_MOVE_SCRATCH;
1435                         if (rval == 0)
1436                                 val = offsetof(struct pt_regs, b0);
1437                         else if (rval == 6)
1438                                 val = offsetof(struct pt_regs, b6);
1439                         else
1440                                 val = offsetof(struct pt_regs, b7);
1441                 }
1442                 break;
1443
1444               case UNW_WHERE_SPREL:
1445                 opc = UNW_INSN_ADD_SP;
1446                 break;
1447
1448               case UNW_WHERE_PSPREL:
1449                 opc = UNW_INSN_ADD_PSP;
1450                 break;
1451
1452               default:
1453                 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1454                            __FUNCTION__, i, r->where);
1455                 break;
1456         }
1457         insn.opc = opc;
1458         insn.dst = unw.preg_index[i];
1459         insn.val = val;
1460         script_emit(script, insn);
1461         if (need_nat_info)
1462                 emit_nat_info(sr, i, script);
1463
1464         if (i == UNW_REG_PSP) {
1465                 /*
1466                  * info->psp must contain the _value_ of the previous
1467                  * sp, not it's save location.  We get this by
1468                  * dereferencing the value we just stored in
1469                  * info->psp:
1470                  */
1471                 insn.opc = UNW_INSN_LOAD;
1472                 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1473                 script_emit(script, insn);
1474         }
1475 }
1476
1477 static inline const struct unw_table_entry *
1478 lookup (struct unw_table *table, unsigned long rel_ip)
1479 {
1480         const struct unw_table_entry *e = 0;
1481         unsigned long lo, hi, mid;
1482
1483         /* do a binary search for right entry: */
1484         for (lo = 0, hi = table->length; lo < hi; ) {
1485                 mid = (lo + hi) / 2;
1486                 e = &table->array[mid];
1487                 if (rel_ip < e->start_offset)
1488                         hi = mid;
1489                 else if (rel_ip >= e->end_offset)
1490                         lo = mid + 1;
1491                 else
1492                         break;
1493         }
1494         if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1495                 return NULL;
1496         return e;
1497 }
1498
1499 /*
1500  * Build an unwind script that unwinds from state OLD_STATE to the
1501  * entrypoint of the function that called OLD_STATE.
1502  */
1503 static inline struct unw_script *
1504 build_script (struct unw_frame_info *info)
1505 {
1506         const struct unw_table_entry *e = 0;
1507         struct unw_script *script = 0;
1508         struct unw_labeled_state *ls, *next;
1509         unsigned long ip = info->ip;
1510         struct unw_state_record sr;
1511         struct unw_table *table;
1512         struct unw_reg_info *r;
1513         struct unw_insn insn;
1514         u8 *dp, *desc_end;
1515         u64 hdr;
1516         int i;
1517         STAT(unsigned long start, parse_start;)
1518
1519         STAT(++unw.stat.script.builds; start = ia64_get_itc());
1520
1521         /* build state record */
1522         memset(&sr, 0, sizeof(sr));
1523         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1524                 r->when = UNW_WHEN_NEVER;
1525         sr.pr_val = info->pr;
1526
1527         UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1528         script = script_new(ip);
1529         if (!script) {
1530                 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __FUNCTION__);
1531                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1532                 return 0;
1533         }
1534         unw.cache[info->prev_script].hint = script - unw.cache;
1535
1536         /* search the kernels and the modules' unwind tables for IP: */
1537
1538         STAT(parse_start = ia64_get_itc());
1539
1540         for (table = unw.tables; table; table = table->next) {
1541                 if (ip >= table->start && ip < table->end) {
1542                         e = lookup(table, ip - table->segment_base);
1543                         break;
1544                 }
1545         }
1546         if (!e) {
1547                 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1548                 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1549                         __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1550                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1551                 sr.curr.reg[UNW_REG_RP].when = -1;
1552                 sr.curr.reg[UNW_REG_RP].val = 0;
1553                 compile_reg(&sr, UNW_REG_RP, script);
1554                 script_finalize(script, &sr);
1555                 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1556                 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1557                 return script;
1558         }
1559
1560         sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1561                           + (ip & 0xfUL));
1562         hdr = *(u64 *) (table->segment_base + e->info_offset);
1563         dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1564         desc_end = dp + 8*UNW_LENGTH(hdr);
1565
1566         while (!sr.done && dp < desc_end)
1567                 dp = unw_decode(dp, sr.in_body, &sr);
1568
1569         if (sr.when_target > sr.epilogue_start) {
1570                 /*
1571                  * sp has been restored and all values on the memory stack below
1572                  * psp also have been restored.
1573                  */
1574                 sr.curr.reg[UNW_REG_PSP].val = 0;
1575                 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1576                 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1577                 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1578                         if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1579                             || r->where == UNW_WHERE_SPREL)
1580                         {
1581                                 r->val = 0;
1582                                 r->where = UNW_WHERE_NONE;
1583                                 r->when = UNW_WHEN_NEVER;
1584                         }
1585         }
1586
1587         script->flags = sr.flags;
1588
1589         /*
1590          * If RP did't get saved, generate entry for the return link
1591          * register.
1592          */
1593         if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1594                 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1595                 sr.curr.reg[UNW_REG_RP].when = -1;
1596                 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1597                 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1598                            __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1599                            sr.curr.reg[UNW_REG_RP].val);
1600         }
1601
1602 #ifdef UNW_DEBUG
1603         UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1604                 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1605         for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1606                 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1607                         UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1608                         switch (r->where) {
1609                               case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1610                               case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1611                               case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1612                               case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1613                               case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1614                               case UNW_WHERE_NONE:
1615                                 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1616                                 break;
1617
1618                               default:
1619                                 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1620                                 break;
1621                         }
1622                         UNW_DPRINT(1, "\t\t%d\n", r->when);
1623                 }
1624         }
1625 #endif
1626
1627         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1628
1629         /* translate state record into unwinder instructions: */
1630
1631         /*
1632          * First, set psp if we're dealing with a fixed-size frame;
1633          * subsequent instructions may depend on this value.
1634          */
1635         if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1636             && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1637             && sr.curr.reg[UNW_REG_PSP].val != 0) {
1638                 /* new psp is sp plus frame size */
1639                 insn.opc = UNW_INSN_ADD;
1640                 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1641                 insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1642                 script_emit(script, insn);
1643         }
1644
1645         /* determine where the primary UNaT is: */
1646         if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1647                 i = UNW_REG_PRI_UNAT_MEM;
1648         else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1649                 i = UNW_REG_PRI_UNAT_GR;
1650         else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1651                 i = UNW_REG_PRI_UNAT_MEM;
1652         else
1653                 i = UNW_REG_PRI_UNAT_GR;
1654
1655         compile_reg(&sr, i, script);
1656
1657         for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1658                 compile_reg(&sr, i, script);
1659
1660         /* free labeled register states & stack: */
1661
1662         STAT(parse_start = ia64_get_itc());
1663         for (ls = sr.labeled_states; ls; ls = next) {
1664                 next = ls->next;
1665                 free_state_stack(&ls->saved_state);
1666                 free_labeled_state(ls);
1667         }
1668         free_state_stack(&sr.curr);
1669         STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1670
1671         script_finalize(script, &sr);
1672         STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1673         return script;
1674 }
1675
1676 /*
1677  * Apply the unwinding actions represented by OPS and update SR to
1678  * reflect the state that existed upon entry to the function that this
1679  * unwinder represents.
1680  */
1681 static inline void
1682 run_script (struct unw_script *script, struct unw_frame_info *state)
1683 {
1684         struct unw_insn *ip, *limit, next_insn;
1685         unsigned long opc, dst, val, off;
1686         unsigned long *s = (unsigned long *) state;
1687         STAT(unsigned long start;)
1688
1689         STAT(++unw.stat.script.runs; start = ia64_get_itc());
1690         state->flags = script->flags;
1691         ip = script->insn;
1692         limit = script->insn + script->count;
1693         next_insn = *ip;
1694
1695         while (ip++ < limit) {
1696                 opc = next_insn.opc;
1697                 dst = next_insn.dst;
1698                 val = next_insn.val;
1699                 next_insn = *ip;
1700
1701           redo:
1702                 switch (opc) {
1703                       case UNW_INSN_ADD:
1704                         s[dst] += val;
1705                         break;
1706
1707                       case UNW_INSN_MOVE2:
1708                         if (!s[val])
1709                                 goto lazy_init;
1710                         s[dst+1] = s[val+1];
1711                         s[dst] = s[val];
1712                         break;
1713
1714                       case UNW_INSN_MOVE:
1715                         if (!s[val])
1716                                 goto lazy_init;
1717                         s[dst] = s[val];
1718                         break;
1719
1720                       case UNW_INSN_MOVE_SCRATCH:
1721                         if (state->pt) {
1722                                 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1723                         } else {
1724                                 s[dst] = 0;
1725                                 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1726                                            __FUNCTION__, dst, val);
1727                         }
1728                         break;
1729
1730                       case UNW_INSN_MOVE_STACKED:
1731                         s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1732                                                                     val);
1733                         break;
1734
1735                       case UNW_INSN_ADD_PSP:
1736                         s[dst] = state->psp + val;
1737                         break;
1738
1739                       case UNW_INSN_ADD_SP:
1740                         s[dst] = state->sp + val;
1741                         break;
1742
1743                       case UNW_INSN_SETNAT_MEMSTK:
1744                         if (!state->pri_unat_loc)
1745                                 state->pri_unat_loc = &state->sw->ar_unat;
1746                         /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1747                         s[dst+1] = (*state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1748                         break;
1749
1750                       case UNW_INSN_SETNAT_TYPE:
1751                         s[dst+1] = val;
1752                         break;
1753
1754                       case UNW_INSN_LOAD:
1755 #ifdef UNW_DEBUG
1756                         if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1757                             || s[val] < TASK_SIZE)
1758                         {
1759                                 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1760                                            __FUNCTION__, s[val]);
1761                                 break;
1762                         }
1763 #endif
1764                         s[dst] = *(unsigned long *) s[val];
1765                         break;
1766                 }
1767         }
1768         STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1769         return;
1770
1771   lazy_init:
1772         off = unw.sw_off[val];
1773         s[val] = (unsigned long) state->sw + off;
1774         if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1775                 /*
1776                  * We're initializing a general register: init NaT info, too.  Note that
1777                  * the offset is a multiple of 8 which gives us the 3 bits needed for
1778                  * the type field.
1779                  */
1780                 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1781         goto redo;
1782 }
1783
1784 static int
1785 find_save_locs (struct unw_frame_info *info)
1786 {
1787         int have_write_lock = 0;
1788         struct unw_script *scr;
1789
1790         if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1791                 /* don't let obviously bad addresses pollute the cache */
1792                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1793                 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1794                 info->rp_loc = 0;
1795                 return -1;
1796         }
1797
1798         scr = script_lookup(info);
1799         if (!scr) {
1800                 scr = build_script(info);
1801                 if (!scr) {
1802                         UNW_DPRINT(0,
1803                                    "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1804                                    __FUNCTION__, info->ip);
1805                         return -1;
1806                 }
1807                 have_write_lock = 1;
1808         }
1809         info->hint = scr->hint;
1810         info->prev_script = scr - unw.cache;
1811
1812         run_script(scr, info);
1813
1814         if (have_write_lock)
1815                 write_unlock(&scr->lock);
1816         else
1817                 read_unlock(&scr->lock);
1818         return 0;
1819 }
1820
1821 int
1822 unw_unwind (struct unw_frame_info *info)
1823 {
1824         unsigned long prev_ip, prev_sp, prev_bsp;
1825         unsigned long ip, pr, num_regs;
1826         STAT(unsigned long start, flags;)
1827         int retval;
1828
1829         STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1830
1831         prev_ip = info->ip;
1832         prev_sp = info->sp;
1833         prev_bsp = info->bsp;
1834
1835         /* restore the ip */
1836         if (!info->rp_loc) {
1837                 /* FIXME: should really be level 0 but it occurs too often. KAO */
1838                 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1839                            __FUNCTION__, info->ip);
1840                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1841                 return -1;
1842         }
1843         ip = info->ip = *info->rp_loc;
1844         if (ip < GATE_ADDR) {
1845                 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1846                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1847                 return -1;
1848         }
1849
1850         /* restore the cfm: */
1851         if (!info->pfs_loc) {
1852                 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1853                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1854                 return -1;
1855         }
1856         info->cfm_loc = info->pfs_loc;
1857
1858         /* restore the bsp: */
1859         pr = info->pr;
1860         num_regs = 0;
1861         if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1862                 info->pt = info->sp + 16;
1863                 if ((pr & (1UL << pNonSys)) != 0)
1864                         num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1865                 info->pfs_loc =
1866                         (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1867                 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1868         } else
1869                 num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1870         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1871         if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1872                 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1873                         __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1874                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1875                 return -1;
1876         }
1877
1878         /* restore the sp: */
1879         info->sp = info->psp;
1880         if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1881                 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1882                         __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1883                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1884                 return -1;
1885         }
1886
1887         if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1888                 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1889                            __FUNCTION__, ip);
1890                 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1891                 return -1;
1892         }
1893
1894         /* as we unwind, the saved ar.unat becomes the primary unat: */
1895         info->pri_unat_loc = info->unat_loc;
1896
1897         /* finally, restore the predicates: */
1898         unw_get_pr(info, &info->pr);
1899
1900         retval = find_save_locs(info);
1901         STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1902         return retval;
1903 }
1904
1905 int
1906 unw_unwind_to_user (struct unw_frame_info *info)
1907 {
1908         unsigned long ip;
1909
1910         while (unw_unwind(info) >= 0) {
1911                 if (unw_get_rp(info, &ip) < 0) {
1912                         unw_get_ip(info, &ip);
1913                         UNW_DPRINT(0, "unwind.%s: failed to read return pointer (ip=0x%lx)\n",
1914                                    __FUNCTION__, ip);
1915                         return -1;
1916                 }
1917                 if (ip < GATE_ADDR)
1918                         return 0;
1919         }
1920         unw_get_ip(info, &ip);
1921         UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n", __FUNCTION__, ip);
1922         return -1;
1923 }
1924
1925 static void
1926 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1927                  struct switch_stack *sw, unsigned long stktop)
1928 {
1929         unsigned long rbslimit, rbstop, stklimit;
1930         STAT(unsigned long start, flags;)
1931
1932         STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1933
1934         /*
1935          * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1936          * don't want to do that because it would be slow as each preserved register would
1937          * have to be processed.  Instead, what we do here is zero out the frame info and
1938          * start the unwind process at the function that created the switch_stack frame.
1939          * When a preserved value in switch_stack needs to be accessed, run_script() will
1940          * initialize the appropriate pointer on demand.
1941          */
1942         memset(info, 0, sizeof(*info));
1943
1944         rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1945         rbstop   = sw->ar_bspstore;
1946         if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1947                 rbstop = rbslimit;
1948
1949         stklimit = (unsigned long) t + IA64_STK_OFFSET;
1950         if (stktop <= rbstop)
1951                 stktop = rbstop;
1952
1953         info->regstk.limit = rbslimit;
1954         info->regstk.top   = rbstop;
1955         info->memstk.limit = stklimit;
1956         info->memstk.top   = stktop;
1957         info->task = t;
1958         info->sw  = sw;
1959         info->sp = info->psp = stktop;
1960         info->pr = sw->pr;
1961         UNW_DPRINT(3, "unwind.%s:\n"
1962                    "  task   0x%lx\n"
1963                    "  rbs = [0x%lx-0x%lx)\n"
1964                    "  stk = [0x%lx-0x%lx)\n"
1965                    "  pr     0x%lx\n"
1966                    "  sw     0x%lx\n"
1967                    "  sp     0x%lx\n",
1968                    __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
1969                    info->pr, (unsigned long) info->sw, info->sp);
1970         STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
1971 }
1972
1973 void
1974 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
1975                             struct pt_regs *pt, struct switch_stack *sw)
1976 {
1977         unsigned long sof;
1978
1979         init_frame_info(info, t, sw, pt->r12);
1980         info->cfm_loc = &pt->cr_ifs;
1981         info->unat_loc = &pt->ar_unat;
1982         info->pfs_loc = &pt->ar_pfs;
1983         sof = *info->cfm_loc & 0x7f;
1984         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
1985         info->ip = pt->cr_iip + ia64_psr(pt)->ri;
1986         info->pt = (unsigned long) pt;
1987         UNW_DPRINT(3, "unwind.%s:\n"
1988                    "  bsp    0x%lx\n"
1989                    "  sof    0x%lx\n"
1990                    "  ip     0x%lx\n",
1991                    __FUNCTION__, info->bsp, sof, info->ip);
1992         find_save_locs(info);
1993 }
1994
1995 void
1996 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
1997 {
1998         unsigned long sol;
1999
2000         init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2001         info->cfm_loc = &sw->ar_pfs;
2002         sol = (*info->cfm_loc >> 7) & 0x7f;
2003         info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2004         info->ip = sw->b0;
2005         UNW_DPRINT(3, "unwind.%s:\n"
2006                    "  bsp    0x%lx\n"
2007                    "  sol    0x%lx\n"
2008                    "  ip     0x%lx\n",
2009                    __FUNCTION__, info->bsp, sol, info->ip);
2010         find_save_locs(info);
2011 }
2012
2013 void
2014 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2015 {
2016         struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2017
2018         UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
2019         unw_init_frame_info(info, t, sw);
2020 }
2021
2022 static void
2023 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2024                    unsigned long gp, const void *table_start, const void *table_end)
2025 {
2026         const struct unw_table_entry *start = table_start, *end = table_end;
2027
2028         table->name = name;
2029         table->segment_base = segment_base;
2030         table->gp = gp;
2031         table->start = segment_base + start[0].start_offset;
2032         table->end = segment_base + end[-1].end_offset;
2033         table->array = start;
2034         table->length = end - start;
2035 }
2036
2037 void *
2038 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2039                       const void *table_start, const void *table_end)
2040 {
2041         const struct unw_table_entry *start = table_start, *end = table_end;
2042         struct unw_table *table;
2043         unsigned long flags;
2044
2045         if (end - start <= 0) {
2046                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2047                            __FUNCTION__);
2048                 return 0;
2049         }
2050
2051         table = kmalloc(sizeof(*table), GFP_USER);
2052         if (!table)
2053                 return 0;
2054
2055         init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2056
2057         spin_lock_irqsave(&unw.lock, flags);
2058         {
2059                 /* keep kernel unwind table at the front (it's searched most commonly): */
2060                 table->next = unw.tables->next;
2061                 unw.tables->next = table;
2062         }
2063         spin_unlock_irqrestore(&unw.lock, flags);
2064
2065         return table;
2066 }
2067
2068 void
2069 unw_remove_unwind_table (void *handle)
2070 {
2071         struct unw_table *table, *prev;
2072         struct unw_script *tmp;
2073         unsigned long flags;
2074         long index;
2075
2076         if (!handle) {
2077                 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2078                            __FUNCTION__);
2079                 return;
2080         }
2081
2082         table = handle;
2083         if (table == &unw.kernel_table) {
2084                 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2085                            "no-can-do!\n", __FUNCTION__);
2086                 return;
2087         }
2088
2089         spin_lock_irqsave(&unw.lock, flags);
2090         {
2091                 /* first, delete the table: */
2092
2093                 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2094                         if (prev->next == table)
2095                                 break;
2096                 if (!prev) {
2097                         UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2098                                    __FUNCTION__, (void *) table);
2099                         spin_unlock_irqrestore(&unw.lock, flags);
2100                         return;
2101                 }
2102                 prev->next = table->next;
2103         }
2104         spin_unlock_irqrestore(&unw.lock, flags);
2105
2106         /* next, remove hash table entries for this table */
2107
2108         for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2109                 tmp = unw.cache + unw.hash[index];
2110                 if (unw.hash[index] >= UNW_CACHE_SIZE
2111                     || tmp->ip < table->start || tmp->ip >= table->end)
2112                         continue;
2113
2114                 write_lock(&tmp->lock);
2115                 {
2116                         if (tmp->ip >= table->start && tmp->ip < table->end) {
2117                                 unw.hash[index] = tmp->coll_chain;
2118                                 tmp->ip = 0;
2119                         }
2120                 }
2121                 write_unlock(&tmp->lock);
2122         }
2123
2124         kfree(table);
2125 }
2126
2127 static void __init
2128 create_gate_table (void)
2129 {
2130         const struct unw_table_entry *entry, *start, *end;
2131         unsigned long *lp, segbase = GATE_ADDR;
2132         size_t info_size, size;
2133         char *info;
2134         Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2135         int i;
2136
2137         for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2138                 if (phdr->p_type == PT_IA_64_UNWIND) {
2139                         punw = phdr;
2140                         break;
2141                 }
2142
2143         if (!punw) {
2144                 printk("%s: failed to find gate DSO's unwind table!\n", __FUNCTION__);
2145                 return;
2146         }
2147
2148         start = (const struct unw_table_entry *) punw->p_vaddr;
2149         end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2150         size  = 0;
2151
2152         unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2153
2154         for (entry = start; entry < end; ++entry)
2155                 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2156         size += 8;      /* reserve space for "end of table" marker */
2157
2158         unw.gate_table = kmalloc(size, GFP_KERNEL);
2159         if (!unw.gate_table) {
2160                 unw.gate_table_size = 0;
2161                 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __FUNCTION__);
2162                 return;
2163         }
2164         unw.gate_table_size = size;
2165
2166         lp = unw.gate_table;
2167         info = (char *) unw.gate_table + size;
2168
2169         for (entry = start; entry < end; ++entry, lp += 3) {
2170                 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2171                 info -= info_size;
2172                 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2173
2174                 lp[0] = segbase + entry->start_offset;          /* start */
2175                 lp[1] = segbase + entry->end_offset;            /* end */
2176                 lp[2] = info - (char *) unw.gate_table;         /* info */
2177         }
2178         *lp = 0;        /* end-of-table marker */
2179 }
2180
2181 __initcall(create_gate_table);
2182
2183 void __init
2184 unw_init (void)
2185 {
2186         extern int ia64_unw_start, ia64_unw_end, __gp;
2187         extern void unw_hash_index_t_is_too_narrow (void);
2188         long i, off;
2189
2190         if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2191                 unw_hash_index_t_is_too_narrow();
2192
2193         unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2194         unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2195         unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2196         unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2197         unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2198         unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2199         unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2200         unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2201         for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2202                 unw.sw_off[unw.preg_index[i]] = off;
2203         for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2204                 unw.sw_off[unw.preg_index[i]] = off;
2205         for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2206                 unw.sw_off[unw.preg_index[i]] = off;
2207         for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2208                 unw.sw_off[unw.preg_index[i]] = off;
2209
2210         for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2211                 if (i > 0)
2212                         unw.cache[i].lru_chain = (i - 1);
2213                 unw.cache[i].coll_chain = -1;
2214                 unw.cache[i].lock = RW_LOCK_UNLOCKED;
2215         }
2216         unw.lru_head = UNW_CACHE_SIZE - 1;
2217         unw.lru_tail = 0;
2218
2219         init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) &__gp,
2220                           &ia64_unw_start, &ia64_unw_end);
2221 }
2222
2223 /*
2224  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2225  *
2226  *      This system call has been deprecated.  The new and improved way to get
2227  *      at the kernel's unwind info is via the gate DSO.  The address of the
2228  *      ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2229  *
2230  * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2231  *
2232  * This system call copies the unwind data into the buffer pointed to by BUF and returns
2233  * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2234  * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2235  * unwind data.
2236  *
2237  * The first portion of the unwind data contains an unwind table and rest contains the
2238  * associated unwind info (in no particular order).  The unwind table consists of a table
2239  * of entries of the form:
2240  *
2241  *      u64 start;      (64-bit address of start of function)
2242  *      u64 end;        (64-bit address of start of function)
2243  *      u64 info;       (BUF-relative offset to unwind info)
2244  *
2245  * The end of the unwind table is indicated by an entry with a START address of zero.
2246  *
2247  * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2248  * on the format of the unwind info.
2249  *
2250  * ERRORS
2251  *      EFAULT  BUF points outside your accessible address space.
2252  */
2253 asmlinkage long
2254 sys_getunwind (void *buf, size_t buf_size)
2255 {
2256         if (buf && buf_size >= unw.gate_table_size)
2257                 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2258                         return -EFAULT;
2259         return unw.gate_table_size;
2260 }