arch/i386/lib/usercopy.c

   1 /*
   2  * User address space access functions.
   3  * The non inlined parts of asm-i386/uaccess.h are here.
   4  *
   5  * Copyright 1997 Andi Kleen <ak@muc.de>
   6  * Copyright 1997 Linus Torvalds
   7  */
   8 #include <linux/config.h>
   9 #include <linux/mm.h>
  10 #include <linux/highmem.h>
  11 #include <linux/blkdev.h>
  12 #include <asm/uaccess.h>
  13 #include <asm/mmx.h>
  14
  15 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
  16 {
  17 #ifdef CONFIG_X86_INTEL_USERCOPY
  18         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
  19                 return 0;
  20 #endif
  21         return 1;
  22 }
  23 #define movsl_is_ok(a1,a2,n) \
  24         __movsl_is_ok((unsigned long)(a1),(unsigned long)(a2),(n))
  25
  26 /*
  27  * Copy a null terminated string from userspace.
  28  */
  29
  30 #define __do_strncpy_from_user(dst,src,count,res)                          \
  31 do {                                                                       \
  32         int __d0, __d1, __d2;                                              \
  33         __asm__ __volatile__(                                              \
  34                 "       testl %1,%1\n"                                     \
  35                 "       jz 2f\n"                                           \
  36                 "0:     lodsb\n"                                           \
  37                 "       stosb\n"                                           \
  38                 "       testb %%al,%%al\n"                                 \
  39                 "       jz 1f\n"                                           \
  40                 "       decl %1\n"                                         \
  41                 "       jnz 0b\n"                                          \
  42                 "1:     subl %1,%0\n"                                      \
  43                 "2:\n"                                                     \
  44                 ".section .fixup,\"ax\"\n"                                 \
  45                 "3:     movl %5,%0\n"                                      \
  46                 "       jmp 2b\n"                                          \
  47                 ".previous\n"                                              \
  48                 ".section __ex_table,\"a\"\n"                              \
  49                 "       .align 4\n"                                        \
  50                 "       .long 0b,3b\n"                                     \
  51                 ".previous"                                                \
  52                 : "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1),      \
  53                   "=&D" (__d2)                                             \
  54                 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
  55                 : "memory");                                               \
  56 } while (0)
  57
  58 /**
  59  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
  60  * @dst:   Destination address, in kernel space.  This buffer must be at
  61  *         least @count bytes long.
  62  * @src:   Source address, in user space.
  63  * @count: Maximum number of bytes to copy, including the trailing NUL.
  64  *
  65  * Copies a NUL-terminated string from userspace to kernel space.
  66  * Caller must check the specified block with access_ok() before calling
  67  * this function.
  68  *
  69  * On success, returns the length of the string (not including the trailing
  70  * NUL).
  71  *
  72  * If access to userspace fails, returns -EFAULT (some data may have been
  73  * copied).
  74  *
  75  * If @count is smaller than the length of the string, copies @count bytes
  76  * and returns @count.
  77  */
  78 long
  79 __strncpy_from_user(char *dst, const char __user *src, long count)
  80 {
  81         long res;
  82         __do_strncpy_from_user(dst, src, count, res);
  83         return res;
  84 }
  85
  86 /**
  87  * strncpy_from_user: - Copy a NUL terminated string from userspace.
  88  * @dst:   Destination address, in kernel space.  This buffer must be at
  89  *         least @count bytes long.
  90  * @src:   Source address, in user space.
  91  * @count: Maximum number of bytes to copy, including the trailing NUL.
  92  *
  93  * Copies a NUL-terminated string from userspace to kernel space.
  94  *
  95  * On success, returns the length of the string (not including the trailing
  96  * NUL).
  97  *
  98  * If access to userspace fails, returns -EFAULT (some data may have been
  99  * copied).
 100  *
 101  * If @count is smaller than the length of the string, copies @count bytes
 102  * and returns @count.
 103  */
 104 long
 105 strncpy_from_user(char *dst, const char __user *src, long count)
 106 {
 107         long res = -EFAULT;
 108         if (access_ok(VERIFY_READ, src, 1))
 109                 __do_strncpy_from_user(dst, src, count, res);
 110         return res;
 111 }
 112
 113
 114 /*
 115  * Zero Userspace
 116  */
 117
 118 #define __do_clear_user(addr,size)                                      \
 119 do {                                                                    \
 120         int __d0;                                                       \
 121         __asm__ __volatile__(                                           \
 122                 "0:     rep; stosl\n"                                   \
 123                 "       movl %2,%0\n"                                   \
 124                 "1:     rep; stosb\n"                                   \
 125                 "2:\n"                                                  \
 126                 ".section .fixup,\"ax\"\n"                              \
 127                 "3:     lea 0(%2,%0,4),%0\n"                            \
 128                 "       jmp 2b\n"                                       \
 129                 ".previous\n"                                           \
 130                 ".section __ex_table,\"a\"\n"                           \
 131                 "       .align 4\n"                                     \
 132                 "       .long 0b,3b\n"                                  \
 133                 "       .long 1b,2b\n"                                  \
 134                 ".previous"                                             \
 135                 : "=&c"(size), "=&D" (__d0)                             \
 136                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
 137 } while (0)
 138
 139 /**
 140  * clear_user: - Zero a block of memory in user space.
 141  * @to:   Destination address, in user space.
 142  * @n:    Number of bytes to zero.
 143  *
 144  * Zero a block of memory in user space.
 145  *
 146  * Returns number of bytes that could not be cleared.
 147  * On success, this will be zero.
 148  */
 149 unsigned long
 150 clear_user(void __user *to, unsigned long n)
 151 {
 152         might_sleep();
 153         if (access_ok(VERIFY_WRITE, to, n))
 154                 __do_clear_user(to, n);
 155         return n;
 156 }
 157
 158 /**
 159  * __clear_user: - Zero a block of memory in user space, with less checking.
 160  * @to:   Destination address, in user space.
 161  * @n:    Number of bytes to zero.
 162  *
 163  * Zero a block of memory in user space.  Caller must check
 164  * the specified block with access_ok() before calling this function.
 165  *
 166  * Returns number of bytes that could not be cleared.
 167  * On success, this will be zero.
 168  */
 169 unsigned long
 170 __clear_user(void __user *to, unsigned long n)
 171 {
 172         __do_clear_user(to, n);
 173         return n;
 174 }
 175
 176 /**
 177  * strlen_user: - Get the size of a string in user space.
 178  * @s: The string to measure.
 179  * @n: The maximum valid length
 180  *
 181  * Get the size of a NUL-terminated string in user space.
 182  *
 183  * Returns the size of the string INCLUDING the terminating NUL.
 184  * On exception, returns 0.
 185  * If the string is too long, returns a value greater than @n.
 186  */
 187 long strnlen_user(const char __user *s, long n)
 188 {
 189         unsigned long mask = -__addr_ok(s);
 190         unsigned long res, tmp;
 191
 192         might_sleep();
 193
 194         __asm__ __volatile__(
 195                 "       testl %0, %0\n"
 196                 "       jz 3f\n"
 197                 "       andl %0,%%ecx\n"
 198                 "0:     repne; scasb\n"
 199                 "       setne %%al\n"
 200                 "       subl %%ecx,%0\n"
 201                 "       addl %0,%%eax\n"
 202                 "1:\n"
 203                 ".section .fixup,\"ax\"\n"
 204                 "2:     xorl %%eax,%%eax\n"
 205                 "       jmp 1b\n"
 206                 "3:     movb $1,%%al\n"
 207                 "       jmp 1b\n"
 208                 ".previous\n"
 209                 ".section __ex_table,\"a\"\n"
 210                 "       .align 4\n"
 211                 "       .long 0b,2b\n"
 212                 ".previous"
 213                 :"=r" (n), "=D" (s), "=a" (res), "=c" (tmp)
 214                 :"0" (n), "1" (s), "2" (0), "3" (mask)
 215                 :"cc");
 216         return res & mask;
 217 }
 218
 219 #ifdef CONFIG_X86_INTEL_USERCOPY
 220 static unsigned long
 221 __copy_user_intel(void *to, const void *from,unsigned long size)
 222 {
 223         int d0, d1;
 224         __asm__ __volatile__(
 225                        "       .align 2,0x90\n"
 226                        "0:     movl 32(%4), %%eax\n"
 227                        "       cmpl $67, %0\n"
 228                        "       jbe 1f\n"
 229                        "       movl 64(%4), %%eax\n"
 230                        "       .align 2,0x90\n"
 231                        "1:     movl 0(%4), %%eax\n"
 232                        "       movl 4(%4), %%edx\n"
 233                        "2:     movl %%eax, 0(%3)\n"
 234                        "21:    movl %%edx, 4(%3)\n"
 235                        "       movl 8(%4), %%eax\n"
 236                        "       movl 12(%4),%%edx\n"
 237                        "3:     movl %%eax, 8(%3)\n"
 238                        "31:    movl %%edx, 12(%3)\n"
 239                        "       movl 16(%4), %%eax\n"
 240                        "       movl 20(%4), %%edx\n"
 241                        "4:     movl %%eax, 16(%3)\n"
 242                        "41:    movl %%edx, 20(%3)\n"
 243                        "       movl 24(%4), %%eax\n"
 244                        "       movl 28(%4), %%edx\n"
 245                        "10:    movl %%eax, 24(%3)\n"
 246                        "51:    movl %%edx, 28(%3)\n"
 247                        "       movl 32(%4), %%eax\n"
 248                        "       movl 36(%4), %%edx\n"
 249                        "11:    movl %%eax, 32(%3)\n"
 250                        "61:    movl %%edx, 36(%3)\n"
 251                        "       movl 40(%4), %%eax\n"
 252                        "       movl 44(%4), %%edx\n"
 253                        "12:    movl %%eax, 40(%3)\n"
 254                        "71:    movl %%edx, 44(%3)\n"
 255                        "       movl 48(%4), %%eax\n"
 256                        "       movl 52(%4), %%edx\n"
 257                        "13:    movl %%eax, 48(%3)\n"
 258                        "81:    movl %%edx, 52(%3)\n"
 259                        "       movl 56(%4), %%eax\n"
 260                        "       movl 60(%4), %%edx\n"
 261                        "14:    movl %%eax, 56(%3)\n"
 262                        "91:    movl %%edx, 60(%3)\n"
 263                        "       addl $-64, %0\n"
 264                        "       addl $64, %4\n"
 265                        "       addl $64, %3\n"
 266                        "       cmpl $63, %0\n"
 267                        "       ja  0b\n"
 268                        "5:     movl  %0, %%eax\n"
 269                        "       shrl  $2, %0\n"
 270                        "       andl  $3, %%eax\n"
 271                        "       cld\n"
 272                        "6:     rep; movsl\n"
 273                        "       movl %%eax, %0\n"
 274                        "7:     rep; movsb\n"
 275                        "8:\n"
 276                        ".section .fixup,\"ax\"\n"
 277                        "9:     lea 0(%%eax,%0,4),%0\n"
 278                        "       jmp 8b\n"
 279                        ".previous\n"
 280                        ".section __ex_table,\"a\"\n"
 281                        "       .align 4\n"
 282                        "       .long 2b,8b\n"
 283                        "       .long 21b,8b\n"
 284                        "       .long 3b,8b\n"
 285                        "       .long 31b,8b\n"
 286                        "       .long 4b,8b\n"
 287                        "       .long 41b,8b\n"
 288                        "       .long 10b,8b\n"
 289                        "       .long 51b,8b\n"
 290                        "       .long 11b,8b\n"
 291                        "       .long 61b,8b\n"
 292                        "       .long 12b,8b\n"
 293                        "       .long 71b,8b\n"
 294                        "       .long 13b,8b\n"
 295                        "       .long 81b,8b\n"
 296                        "       .long 14b,8b\n"
 297                        "       .long 91b,8b\n"
 298                        "       .long 6b,9b\n"
 299                        "       .long 7b,8b\n"
 300                        ".previous"
 301                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 302                        :  "1"(to), "2"(from), "0"(size)
 303                        : "eax", "edx", "memory");
 304         return size;
 305 }
 306
 307 static unsigned long
 308 __copy_user_zeroing_intel(void *to, const void *from, unsigned long size)
 309 {
 310         int d0, d1;
 311         __asm__ __volatile__(
 312                        "        .align 2,0x90\n"
 313                        "0:      movl 32(%4), %%eax\n"
 314                        "        cmpl $67, %0\n"
 315                        "        jbe 2f\n"
 316                        "1:      movl 64(%4), %%eax\n"
 317                        "        .align 2,0x90\n"
 318                        "2:      movl 0(%4), %%eax\n"
 319                        "21:     movl 4(%4), %%edx\n"
 320                        "        movl %%eax, 0(%3)\n"
 321                        "        movl %%edx, 4(%3)\n"
 322                        "3:      movl 8(%4), %%eax\n"
 323                        "31:     movl 12(%4),%%edx\n"
 324                        "        movl %%eax, 8(%3)\n"
 325                        "        movl %%edx, 12(%3)\n"
 326                        "4:      movl 16(%4), %%eax\n"
 327                        "41:     movl 20(%4), %%edx\n"
 328                        "        movl %%eax, 16(%3)\n"
 329                        "        movl %%edx, 20(%3)\n"
 330                        "10:     movl 24(%4), %%eax\n"
 331                        "51:     movl 28(%4), %%edx\n"
 332                        "        movl %%eax, 24(%3)\n"
 333                        "        movl %%edx, 28(%3)\n"
 334                        "11:     movl 32(%4), %%eax\n"
 335                        "61:     movl 36(%4), %%edx\n"
 336                        "        movl %%eax, 32(%3)\n"
 337                        "        movl %%edx, 36(%3)\n"
 338                        "12:     movl 40(%4), %%eax\n"
 339                        "71:     movl 44(%4), %%edx\n"
 340                        "        movl %%eax, 40(%3)\n"
 341                        "        movl %%edx, 44(%3)\n"
 342                        "13:     movl 48(%4), %%eax\n"
 343                        "81:     movl 52(%4), %%edx\n"
 344                        "        movl %%eax, 48(%3)\n"
 345                        "        movl %%edx, 52(%3)\n"
 346                        "14:     movl 56(%4), %%eax\n"
 347                        "91:     movl 60(%4), %%edx\n"
 348                        "        movl %%eax, 56(%3)\n"
 349                        "        movl %%edx, 60(%3)\n"
 350                        "        addl $-64, %0\n"
 351                        "        addl $64, %4\n"
 352                        "        addl $64, %3\n"
 353                        "        cmpl $63, %0\n"
 354                        "        ja  0b\n"
 355                        "5:      movl  %0, %%eax\n"
 356                        "        shrl  $2, %0\n"
 357                        "        andl $3, %%eax\n"
 358                        "        cld\n"
 359                        "6:      rep; movsl\n"
 360                        "        movl %%eax,%0\n"
 361                        "7:      rep; movsb\n"
 362                        "8:\n"
 363                        ".section .fixup,\"ax\"\n"
 364                        "9:      lea 0(%%eax,%0,4),%0\n"
 365                        "16:     pushl %0\n"
 366                        "        pushl %%eax\n"
 367                        "        xorl %%eax,%%eax\n"
 368                        "        rep; stosb\n"
 369                        "        popl %%eax\n"
 370                        "        popl %0\n"
 371                        "        jmp 8b\n"
 372                        ".previous\n"
 373                        ".section __ex_table,\"a\"\n"
 374                        "        .align 4\n"
 375                        "        .long 0b,16b\n"
 376                        "        .long 1b,16b\n"
 377                        "        .long 2b,16b\n"
 378                        "        .long 21b,16b\n"
 379                        "        .long 3b,16b\n"
 380                        "        .long 31b,16b\n"
 381                        "        .long 4b,16b\n"
 382                        "        .long 41b,16b\n"
 383                        "        .long 10b,16b\n"
 384                        "        .long 51b,16b\n"
 385                        "        .long 11b,16b\n"
 386                        "        .long 61b,16b\n"
 387                        "        .long 12b,16b\n"
 388                        "        .long 71b,16b\n"
 389                        "        .long 13b,16b\n"
 390                        "        .long 81b,16b\n"
 391                        "        .long 14b,16b\n"
 392                        "        .long 91b,16b\n"
 393                        "        .long 6b,9b\n"
 394                        "        .long 7b,16b\n"
 395                        ".previous"
 396                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 397                        :  "1"(to), "2"(from), "0"(size)
 398                        : "eax", "edx", "memory");
 399         return size;
 400 }
 401 #else
 402 /*
 403  * Leave these declared but undefined.  They should not be any references to
 404  * them
 405  */
 406 unsigned long
 407 __copy_user_zeroing_intel(void *to, const void *from, unsigned long size);
 408 unsigned long
 409 __copy_user_intel(void *to, const void *from,unsigned long size);
 410 #endif /* CONFIG_X86_INTEL_USERCOPY */
 411
 412 /* Generic arbitrary sized copy.  */
 413 #define __copy_user(to,from,size)                                       \
 414 do {                                                                    \
 415         int __d0, __d1, __d2;                                           \
 416         __asm__ __volatile__(                                           \
 417                 "       cmp  $7,%0\n"                                   \
 418                 "       jbe  1f\n"                                      \
 419                 "       movl %1,%0\n"                                   \
 420                 "       negl %0\n"                                      \
 421                 "       andl $7,%0\n"                                   \
 422                 "       subl %0,%3\n"                                   \
 423                 "4:     rep; movsb\n"                                   \
 424                 "       movl %3,%0\n"                                   \
 425                 "       shrl $2,%0\n"                                   \
 426                 "       andl $3,%3\n"                                   \
 427                 "       .align 2,0x90\n"                                \
 428                 "0:     rep; movsl\n"                                   \
 429                 "       movl %3,%0\n"                                   \
 430                 "1:     rep; movsb\n"                                   \
 431                 "2:\n"                                                  \
 432                 ".section .fixup,\"ax\"\n"                              \
 433                 "5:     addl %3,%0\n"                                   \
 434                 "       jmp 2b\n"                                       \
 435                 "3:     lea 0(%3,%0,4),%0\n"                            \
 436                 "       jmp 2b\n"                                       \
 437                 ".previous\n"                                           \
 438                 ".section __ex_table,\"a\"\n"                           \
 439                 "       .align 4\n"                                     \
 440                 "       .long 4b,5b\n"                                  \
 441                 "       .long 0b,3b\n"                                  \
 442                 "       .long 1b,2b\n"                                  \
 443                 ".previous"                                             \
 444                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 445                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 446                 : "memory");                                            \
 447 } while (0)
 448
 449 #define __copy_user_zeroing(to,from,size)                               \
 450 do {                                                                    \
 451         int __d0, __d1, __d2;                                           \
 452         __asm__ __volatile__(                                           \
 453                 "       cmp  $7,%0\n"                                   \
 454                 "       jbe  1f\n"                                      \
 455                 "       movl %1,%0\n"                                   \
 456                 "       negl %0\n"                                      \
 457                 "       andl $7,%0\n"                                   \
 458                 "       subl %0,%3\n"                                   \
 459                 "4:     rep; movsb\n"                                   \
 460                 "       movl %3,%0\n"                                   \
 461                 "       shrl $2,%0\n"                                   \
 462                 "       andl $3,%3\n"                                   \
 463                 "       .align 2,0x90\n"                                \
 464                 "0:     rep; movsl\n"                                   \
 465                 "       movl %3,%0\n"                                   \
 466                 "1:     rep; movsb\n"                                   \
 467                 "2:\n"                                                  \
 468                 ".section .fixup,\"ax\"\n"                              \
 469                 "5:     addl %3,%0\n"                                   \
 470                 "       jmp 6f\n"                                       \
 471                 "3:     lea 0(%3,%0,4),%0\n"                            \
 472                 "6:     pushl %0\n"                                     \
 473                 "       pushl %%eax\n"                                  \
 474                 "       xorl %%eax,%%eax\n"                             \
 475                 "       rep; stosb\n"                                   \
 476                 "       popl %%eax\n"                                   \
 477                 "       popl %0\n"                                      \
 478                 "       jmp 2b\n"                                       \
 479                 ".previous\n"                                           \
 480                 ".section __ex_table,\"a\"\n"                           \
 481                 "       .align 4\n"                                     \
 482                 "       .long 4b,5b\n"                                  \
 483                 "       .long 0b,3b\n"                                  \
 484                 "       .long 1b,6b\n"                                  \
 485                 ".previous"                                             \
 486                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 487                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 488                 : "memory");                                            \
 489 } while (0)
 490
 491
 492 unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned long n)
 493 {
 494 #ifndef CONFIG_X86_WP_WORKS_OK
 495         if (unlikely(boot_cpu_data.wp_works_ok == 0) &&
 496                         ((unsigned long )to) < TASK_SIZE) {
 497                 /*
 498                  * CPU does not honor the WP bit when writing
 499                  * from supervisory mode, and due to preemption or SMP,
 500                  * the page tables can change at any time.
 501                  * Do it manually.      Manfred <manfred@colorfullife.com>
 502                  */
 503                 while (n) {
 504                         unsigned long offset = ((unsigned long)to)%PAGE_SIZE;
 505                         unsigned long len = PAGE_SIZE - offset;
 506                         int retval;
 507                         struct page *pg;
 508                         void *maddr;
 509
 510                         if (len > n)
 511                                 len = n;
 512
 513 survive:
 514                         down_read(&current->mm->mmap_sem);
 515                         retval = get_user_pages(current, current->mm,
 516                                         (unsigned long )to, 1, 1, 0, &pg, NULL);
 517
 518                         if (retval == -ENOMEM && current->pid == 1) {
 519                                 up_read(&current->mm->mmap_sem);
 520                                 blk_congestion_wait(WRITE, HZ/50);
 521                                 goto survive;
 522                         }
 523
 524                         if (retval != 1)
 525                                 break;
 526
 527                         maddr = kmap_atomic(pg, KM_USER0);
 528                         memcpy(maddr + offset, from, len);
 529                         kunmap_atomic(maddr, KM_USER0);
 530                         set_page_dirty_lock(pg);
 531                         put_page(pg);
 532                         up_read(&current->mm->mmap_sem);
 533
 534                         from += len;
 535                         to += len;
 536                         n -= len;
 537                 }
 538                 return n;
 539         }
 540 #endif
 541         if (movsl_is_ok(to, from, n))
 542                 __copy_user((void *)to, from, n);
 543         else
 544                 n = __copy_user_intel((void *)to, from, n);
 545         return n;
 546 }
 547
 548 unsigned long __copy_from_user_ll(void *to, const void __user *from, unsigned long n)
 549 {
 550         if (movsl_is_ok(to, from, n))
 551                 __copy_user_zeroing(to, (const void *) from, n);
 552         else
 553                 n = __copy_user_zeroing_intel(to, (const void *) from, n);
 554         return n;
 555 }