arch/x86/kernel/sysenter_32.c

   1 /*
   2  * (C) Copyright 2002 Linus Torvalds
   3  * Portions based on the vdso-randomization code from exec-shield:
   4  * Copyright(C) 2005-2006, Red Hat, Inc., Ingo Molnar
   5  *
   6  * This file contains the needed initializations to support sysenter.
   7  */
   8
   9 #include <linux/init.h>
  10 #include <linux/smp.h>
  11 #include <linux/thread_info.h>
  12 #include <linux/sched.h>
  13 #include <linux/gfp.h>
  14 #include <linux/string.h>
  15 #include <linux/elf.h>
  16 #include <linux/mm.h>
  17 #include <linux/err.h>
  18 #include <linux/module.h>
  19
  20 #include <asm/cpufeature.h>
  21 #include <asm/msr.h>
  22 #include <asm/pgtable.h>
  23 #include <asm/unistd.h>
  24 #include <asm/elf.h>
  25 #include <asm/tlbflush.h>
  26
  27 enum {
  28         VDSO_DISABLED = 0,
  29         VDSO_ENABLED = 1,
  30         VDSO_COMPAT = 2,
  31 };
  32
  33 #ifdef CONFIG_COMPAT_VDSO
  34 #define VDSO_DEFAULT    VDSO_COMPAT
  35 #else
  36 #define VDSO_DEFAULT    VDSO_ENABLED
  37 #endif
  38
  39 #ifdef CONFIG_XEN
  40 #include <xen/interface/callback.h>
  41 #endif
  42
  43 /*
  44  * Should the kernel map a VDSO page into processes and pass its
  45  * address down to glibc upon exec()?
  46  */
  47 unsigned int __read_mostly vdso_enabled = VDSO_DEFAULT;
  48
  49 EXPORT_SYMBOL_GPL(vdso_enabled);
  50
  51 static int __init vdso_setup(char *s)
  52 {
  53         vdso_enabled = simple_strtoul(s, NULL, 0);
  54
  55         return 1;
  56 }
  57
  58 __setup("vdso=", vdso_setup);
  59
  60 extern asmlinkage void sysenter_entry(void);
  61
  62 static __init void reloc_symtab(Elf32_Ehdr *ehdr,
  63                                 unsigned offset, unsigned size)
  64 {
  65         Elf32_Sym *sym = (void *)ehdr + offset;
  66         unsigned nsym = size / sizeof(*sym);
  67         unsigned i;
  68
  69         for(i = 0; i < nsym; i++, sym++) {
  70                 if (sym->st_shndx == SHN_UNDEF ||
  71                     sym->st_shndx == SHN_ABS)
  72                         continue;  /* skip */
  73
  74                 if (sym->st_shndx > SHN_LORESERVE) {
  75                         printk(KERN_INFO "VDSO: unexpected st_shndx %x\n",
  76                                sym->st_shndx);
  77                         continue;
  78                 }
  79
  80                 switch(ELF_ST_TYPE(sym->st_info)) {
  81                 case STT_OBJECT:
  82                 case STT_FUNC:
  83                 case STT_SECTION:
  84                 case STT_FILE:
  85                         sym->st_value += VDSO_HIGH_BASE;
  86                 }
  87         }
  88 }
  89
  90 static __init void reloc_dyn(Elf32_Ehdr *ehdr, unsigned offset)
  91 {
  92         Elf32_Dyn *dyn = (void *)ehdr + offset;
  93
  94         for(; dyn->d_tag != DT_NULL; dyn++)
  95                 switch(dyn->d_tag) {
  96                 case DT_PLTGOT:
  97                 case DT_HASH:
  98                 case DT_STRTAB:
  99                 case DT_SYMTAB:
 100                 case DT_RELA:
 101                 case DT_INIT:
 102                 case DT_FINI:
 103                 case DT_REL:
 104                 case DT_DEBUG:
 105                 case DT_JMPREL:
 106                 case DT_VERSYM:
 107                 case DT_VERDEF:
 108                 case DT_VERNEED:
 109                 case DT_ADDRRNGLO ... DT_ADDRRNGHI:
 110                         /* definitely pointers needing relocation */
 111                         dyn->d_un.d_ptr += VDSO_HIGH_BASE;
 112                         break;
 113
 114                 case DT_ENCODING ... OLD_DT_LOOS-1:
 115                 case DT_LOOS ... DT_HIOS-1:
 116                         /* Tags above DT_ENCODING are pointers if
 117                            they're even */
 118                         if (dyn->d_tag >= DT_ENCODING &&
 119                             (dyn->d_tag & 1) == 0)
 120                                 dyn->d_un.d_ptr += VDSO_HIGH_BASE;
 121                         break;
 122
 123                 case DT_VERDEFNUM:
 124                 case DT_VERNEEDNUM:
 125                 case DT_FLAGS_1:
 126                 case DT_RELACOUNT:
 127                 case DT_RELCOUNT:
 128                 case DT_VALRNGLO ... DT_VALRNGHI:
 129                         /* definitely not pointers */
 130                         break;
 131
 132                 case OLD_DT_LOOS ... DT_LOOS-1:
 133                 case DT_HIOS ... DT_VALRNGLO-1:
 134                 default:
 135                         if (dyn->d_tag > DT_ENCODING)
 136                                 printk(KERN_INFO "VDSO: unexpected DT_tag %x\n",
 137                                        dyn->d_tag);
 138                         break;
 139                 }
 140 }
 141
 142 static __init void relocate_vdso(Elf32_Ehdr *ehdr)
 143 {
 144         Elf32_Phdr *phdr;
 145         Elf32_Shdr *shdr;
 146         int i;
 147
 148         BUG_ON(memcmp(ehdr->e_ident, ELFMAG, 4) != 0 ||
 149                !elf_check_arch(ehdr) ||
 150                ehdr->e_type != ET_DYN);
 151
 152         ehdr->e_entry += VDSO_HIGH_BASE;
 153
 154         /* rebase phdrs */
 155         phdr = (void *)ehdr + ehdr->e_phoff;
 156         for (i = 0; i < ehdr->e_phnum; i++) {
 157                 phdr[i].p_vaddr += VDSO_HIGH_BASE;
 158
 159                 /* relocate dynamic stuff */
 160                 if (phdr[i].p_type == PT_DYNAMIC)
 161                         reloc_dyn(ehdr, phdr[i].p_offset);
 162         }
 163
 164         /* rebase sections */
 165         shdr = (void *)ehdr + ehdr->e_shoff;
 166         for(i = 0; i < ehdr->e_shnum; i++) {
 167                 if (!(shdr[i].sh_flags & SHF_ALLOC))
 168                         continue;
 169
 170                 shdr[i].sh_addr += VDSO_HIGH_BASE;
 171
 172                 if (shdr[i].sh_type == SHT_SYMTAB ||
 173                     shdr[i].sh_type == SHT_DYNSYM)
 174                         reloc_symtab(ehdr, shdr[i].sh_offset,
 175                                      shdr[i].sh_size);
 176         }
 177 }
 178
 179 void enable_sep_cpu(void)
 180 {
 181 #ifndef CONFIG_XEN
 182         int cpu = get_cpu();
 183         struct tss_struct *tss = &per_cpu(init_tss, cpu);
 184
 185         if (!boot_cpu_has(X86_FEATURE_SEP)) {
 186                 put_cpu();
 187                 return;
 188         }
 189
 190         tss->x86_tss.ss1 = __KERNEL_CS;
 191         tss->x86_tss.esp1 = sizeof(struct tss_struct) + (unsigned long) tss;
 192         wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
 193         wrmsr(MSR_IA32_SYSENTER_ESP, tss->x86_tss.esp1, 0);
 194         wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long) sysenter_entry, 0);
 195 #else
 196         extern asmlinkage void sysenter_entry_pv(void);
 197         static struct callback_register sysenter = {
 198                 .type = CALLBACKTYPE_sysenter,
 199                 .address = { __KERNEL_CS, (unsigned long)sysenter_entry_pv },
 200         };
 201
 202         if (!boot_cpu_has(X86_FEATURE_SEP))
 203                 return;
 204
 205         get_cpu();
 206
 207         if (xen_feature(XENFEAT_supervisor_mode_kernel))
 208                 sysenter.address.eip = (unsigned long)sysenter_entry;
 209
 210         switch (HYPERVISOR_callback_op(CALLBACKOP_register, &sysenter)) {
 211         case 0:
 212                 break;
 213 #if CONFIG_XEN_COMPAT < 0x030200
 214         case -ENOSYS:
 215                 sysenter.type = CALLBACKTYPE_sysenter_deprecated;
 216                 if (HYPERVISOR_callback_op(CALLBACKOP_register, &sysenter) == 0)
 217                         break;
 218 #endif
 219         default:
 220                 clear_bit(X86_FEATURE_SEP, boot_cpu_data.x86_capability);
 221                 break;
 222         }
 223 #endif
 224         put_cpu();
 225 }
 226
 227 static struct vm_area_struct gate_vma;
 228
 229 static int __init gate_vma_init(void)
 230 {
 231         gate_vma.vm_mm = NULL;
 232         gate_vma.vm_start = FIXADDR_USER_START;
 233         gate_vma.vm_end = FIXADDR_USER_END;
 234         gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
 235         gate_vma.vm_page_prot = __P101;
 236         /*
 237          * Make sure the vDSO gets into every core dump.
 238          * Dumping its contents makes post-mortem fully interpretable later
 239          * without matching up the same kernel and hardware config to see
 240          * what PC values meant.
 241          */
 242         gate_vma.vm_flags |= VM_ALWAYSDUMP;
 243         return 0;
 244 }
 245
 246 /*
 247  * These symbols are defined by vsyscall.o to mark the bounds
 248  * of the ELF DSO images included therein.
 249  */
 250 extern const char vsyscall_int80_start, vsyscall_int80_end;
 251 extern const char vsyscall_sysenter_start, vsyscall_sysenter_end;
 252 static struct page *syscall_pages[1];
 253
 254 static void map_compat_vdso(int map)
 255 {
 256         static int vdso_mapped;
 257
 258         if (map == vdso_mapped)
 259                 return;
 260
 261         vdso_mapped = map;
 262
 263         __set_fixmap(FIX_VDSO, page_to_pfn(syscall_pages[0]) << PAGE_SHIFT,
 264                      map ? PAGE_READONLY_EXEC : PAGE_NONE);
 265
 266         /* flush stray tlbs */
 267         flush_tlb_all();
 268 }
 269
 270 int __init sysenter_setup(void)
 271 {
 272         void *syscall_page = (void *)get_zeroed_page(GFP_ATOMIC);
 273         const void *vsyscall;
 274         size_t vsyscall_len;
 275
 276         syscall_pages[0] = virt_to_page(syscall_page);
 277
 278         gate_vma_init();
 279
 280         printk("Compat vDSO mapped to %08lx.\n", __fix_to_virt(FIX_VDSO));
 281
 282         if (!boot_cpu_has(X86_FEATURE_SEP)) {
 283                 vsyscall = &vsyscall_int80_start;
 284                 vsyscall_len = &vsyscall_int80_end - &vsyscall_int80_start;
 285         } else {
 286                 vsyscall = &vsyscall_sysenter_start;
 287                 vsyscall_len = &vsyscall_sysenter_end - &vsyscall_sysenter_start;
 288         }
 289
 290         memcpy(syscall_page, vsyscall, vsyscall_len);
 291         relocate_vdso(syscall_page);
 292
 293         return 0;
 294 }
 295
 296 /* Defined in vsyscall-sysenter.S */
 297 extern void SYSENTER_RETURN;
 298
 299 /* Setup a VMA at program startup for the vsyscall page */
 300 int arch_setup_additional_pages(struct linux_binprm *bprm, int exstack)
 301 {
 302         struct mm_struct *mm = current->mm;
 303         unsigned long addr;
 304         int ret = 0;
 305         bool compat;
 306
 307         down_write(&mm->mmap_sem);
 308
 309         /* Test compat mode once here, in case someone
 310            changes it via sysctl */
 311         compat = (vdso_enabled == VDSO_COMPAT);
 312
 313         map_compat_vdso(compat);
 314
 315         if (compat)
 316                 addr = VDSO_HIGH_BASE;
 317         else {
 318                 addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
 319                 if (IS_ERR_VALUE(addr)) {
 320                         ret = addr;
 321                         goto up_fail;
 322                 }
 323
 324                 /*
 325                  * MAYWRITE to allow gdb to COW and set breakpoints
 326                  *
 327                  * Make sure the vDSO gets into every core dump.
 328                  * Dumping its contents makes post-mortem fully
 329                  * interpretable later without matching up the same
 330                  * kernel and hardware config to see what PC values
 331                  * meant.
 332                  */
 333                 ret = install_special_mapping(mm, addr, PAGE_SIZE,
 334                                               VM_READ|VM_EXEC|
 335                                               VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC|
 336                                               VM_ALWAYSDUMP,
 337                                               syscall_pages);
 338
 339                 if (ret)
 340                         goto up_fail;
 341         }
 342
 343         current->mm->context.vdso = (void *)addr;
 344         current_thread_info()->sysenter_return =
 345                 (void *)VDSO_SYM(&SYSENTER_RETURN);
 346
 347   up_fail:
 348         up_write(&mm->mmap_sem);
 349
 350         return ret;
 351 }
 352
 353 const char *arch_vma_name(struct vm_area_struct *vma)
 354 {
 355         if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
 356                 return "[vdso]";
 357         return NULL;
 358 }
 359
 360 struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
 361 {
 362         struct mm_struct *mm = tsk->mm;
 363
 364         /* Check to see if this task was created in compat vdso mode */
 365         if (mm && mm->context.vdso == (void *)VDSO_HIGH_BASE)
 366                 return &gate_vma;
 367         return NULL;
 368 }
 369
 370 int in_gate_area(struct task_struct *task, unsigned long addr)
 371 {
 372         const struct vm_area_struct *vma = get_gate_vma(task);
 373
 374         return vma && addr >= vma->vm_start && addr < vma->vm_end;
 375 }
 376
 377 int in_gate_area_no_task(unsigned long addr)
 378 {
 379         return 0;
 380 }