arch/x86/mm/hypervisor.c

   1 /******************************************************************************
   2  * mm/hypervisor.c
   3  *
   4  * Update page tables via the hypervisor.
   5  *
   6  * Copyright (c) 2002-2004, K A Fraser
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License version 2
  10  * as published by the Free Software Foundation; or, when distributed
  11  * separately from the Linux kernel or incorporated into other
  12  * software packages, subject to the following license:
  13  *
  14  * Permission is hereby granted, free of charge, to any person obtaining a copy
  15  * of this source file (the "Software"), to deal in the Software without
  16  * restriction, including without limitation the rights to use, copy, modify,
  17  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  18  * and to permit persons to whom the Software is furnished to do so, subject to
  19  * the following conditions:
  20  *
  21  * The above copyright notice and this permission notice shall be included in
  22  * all copies or substantial portions of the Software.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  25  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  26  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  27  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  28  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  29  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  30  * IN THE SOFTWARE.
  31  */
  32
  33 #include <linux/sched.h>
  34 #include <linux/mm.h>
  35 #include <linux/vmalloc.h>
  36 #include <asm/page.h>
  37 #include <asm/pgtable.h>
  38 #include <asm/hypervisor.h>
  39 #include <xen/balloon.h>
  40 #include <xen/features.h>
  41 #include <xen/interface/memory.h>
  42 #include <linux/module.h>
  43 #include <linux/percpu.h>
  44 #include <linux/highmem.h>
  45 #include <asm/tlbflush.h>
  46 #include <linux/highmem.h>
  47
  48 void xen_l1_entry_update(pte_t *ptr, pte_t val)
  49 {
  50         mmu_update_t u;
  51 #ifdef CONFIG_HIGHPTE
  52         u.ptr = ((unsigned long)ptr >= (unsigned long)high_memory) ?
  53                 arbitrary_virt_to_machine(ptr) : virt_to_machine(ptr);
  54 #else
  55         u.ptr = virt_to_machine(ptr);
  56 #endif
  57         u.val = __pte_val(val);
  58         BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  59 }
  60 EXPORT_SYMBOL_GPL(xen_l1_entry_update);
  61
  62 void xen_l2_entry_update(pmd_t *ptr, pmd_t val)
  63 {
  64         mmu_update_t u;
  65         u.ptr = virt_to_machine(ptr);
  66         u.val = __pmd_val(val);
  67         BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  68 }
  69
  70 #if defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64)
  71 void xen_l3_entry_update(pud_t *ptr, pud_t val)
  72 {
  73         mmu_update_t u;
  74         u.ptr = virt_to_machine(ptr);
  75         u.val = __pud_val(val);
  76         BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  77 }
  78 #endif
  79
  80 #ifdef CONFIG_X86_64
  81 void xen_l4_entry_update(pgd_t *ptr, pgd_t val)
  82 {
  83         mmu_update_t u;
  84         u.ptr = virt_to_machine(ptr);
  85         u.val = __pgd_val(val);
  86         BUG_ON(HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0);
  87 }
  88 #endif /* CONFIG_X86_64 */
  89
  90 void xen_pt_switch(unsigned long ptr)
  91 {
  92         struct mmuext_op op;
  93         op.cmd = MMUEXT_NEW_BASEPTR;
  94         op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
  95         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
  96 }
  97
  98 void xen_new_user_pt(unsigned long ptr)
  99 {
 100         struct mmuext_op op;
 101         op.cmd = MMUEXT_NEW_USER_BASEPTR;
 102         op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
 103         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 104 }
 105
 106 void xen_tlb_flush(void)
 107 {
 108         struct mmuext_op op;
 109         op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
 110         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 111 }
 112 EXPORT_SYMBOL(xen_tlb_flush);
 113
 114 void xen_invlpg(unsigned long ptr)
 115 {
 116         struct mmuext_op op;
 117         op.cmd = MMUEXT_INVLPG_LOCAL;
 118         op.arg1.linear_addr = ptr & PAGE_MASK;
 119         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 120 }
 121 EXPORT_SYMBOL(xen_invlpg);
 122
 123 #ifdef CONFIG_SMP
 124
 125 void xen_tlb_flush_all(void)
 126 {
 127         struct mmuext_op op;
 128         op.cmd = MMUEXT_TLB_FLUSH_ALL;
 129         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 130 }
 131
 132 void xen_tlb_flush_mask(cpumask_t *mask)
 133 {
 134         struct mmuext_op op;
 135         if ( cpus_empty(*mask) )
 136                 return;
 137         op.cmd = MMUEXT_TLB_FLUSH_MULTI;
 138         set_xen_guest_handle(op.arg2.vcpumask, mask->bits);
 139         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 140 }
 141
 142 void xen_invlpg_all(unsigned long ptr)
 143 {
 144         struct mmuext_op op;
 145         op.cmd = MMUEXT_INVLPG_ALL;
 146         op.arg1.linear_addr = ptr & PAGE_MASK;
 147         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 148 }
 149
 150 void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
 151 {
 152         struct mmuext_op op;
 153         if ( cpus_empty(*mask) )
 154                 return;
 155         op.cmd = MMUEXT_INVLPG_MULTI;
 156         op.arg1.linear_addr = ptr & PAGE_MASK;
 157         set_xen_guest_handle(op.arg2.vcpumask, mask->bits);
 158         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 159 }
 160
 161 #endif /* CONFIG_SMP */
 162
 163 void xen_pgd_pin(unsigned long ptr)
 164 {
 165         struct mmuext_op op;
 166 #ifdef CONFIG_X86_64
 167         op.cmd = MMUEXT_PIN_L4_TABLE;
 168 #elif defined(CONFIG_X86_PAE)
 169         op.cmd = MMUEXT_PIN_L3_TABLE;
 170 #else
 171         op.cmd = MMUEXT_PIN_L2_TABLE;
 172 #endif
 173         op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
 174         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 175 }
 176
 177 void xen_pgd_unpin(unsigned long ptr)
 178 {
 179         struct mmuext_op op;
 180         op.cmd = MMUEXT_UNPIN_TABLE;
 181         op.arg1.mfn = pfn_to_mfn(ptr >> PAGE_SHIFT);
 182         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 183 }
 184
 185 void xen_set_ldt(const void *ptr, unsigned int ents)
 186 {
 187         struct mmuext_op op;
 188         op.cmd = MMUEXT_SET_LDT;
 189         op.arg1.linear_addr = (unsigned long)ptr;
 190         op.arg2.nr_ents     = ents;
 191         BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 192 }
 193
 194 /*
 195  * Bitmap is indexed by page number. If bit is set, the page is part of a
 196  * xen_create_contiguous_region() area of memory.
 197  */
 198 unsigned long *contiguous_bitmap;
 199
 200 static void contiguous_bitmap_set(
 201         unsigned long first_page, unsigned long nr_pages)
 202 {
 203         unsigned long start_off, end_off, curr_idx, end_idx;
 204
 205         curr_idx  = first_page / BITS_PER_LONG;
 206         start_off = first_page & (BITS_PER_LONG-1);
 207         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
 208         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
 209
 210         if (curr_idx == end_idx) {
 211                 contiguous_bitmap[curr_idx] |=
 212                         ((1UL<<end_off)-1) & -(1UL<<start_off);
 213         } else {
 214                 contiguous_bitmap[curr_idx] |= -(1UL<<start_off);
 215                 while ( ++curr_idx < end_idx )
 216                         contiguous_bitmap[curr_idx] = ~0UL;
 217                 contiguous_bitmap[curr_idx] |= (1UL<<end_off)-1;
 218         }
 219 }
 220
 221 static void contiguous_bitmap_clear(
 222         unsigned long first_page, unsigned long nr_pages)
 223 {
 224         unsigned long start_off, end_off, curr_idx, end_idx;
 225
 226         curr_idx  = first_page / BITS_PER_LONG;
 227         start_off = first_page & (BITS_PER_LONG-1);
 228         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
 229         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
 230
 231         if (curr_idx == end_idx) {
 232                 contiguous_bitmap[curr_idx] &=
 233                         -(1UL<<end_off) | ((1UL<<start_off)-1);
 234         } else {
 235                 contiguous_bitmap[curr_idx] &= (1UL<<start_off)-1;
 236                 while ( ++curr_idx != end_idx )
 237                         contiguous_bitmap[curr_idx] = 0;
 238                 contiguous_bitmap[curr_idx] &= -(1UL<<end_off);
 239         }
 240 }
 241
 242 /* Protected by balloon_lock. */
 243 #define MAX_CONTIG_ORDER 9 /* 2MB */
 244 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
 245 static unsigned long limited_frames[1<<MAX_CONTIG_ORDER];
 246 static multicall_entry_t cr_mcl[1<<MAX_CONTIG_ORDER];
 247
 248 /* Ensure multi-page extents are contiguous in machine memory. */
 249 int xen_create_contiguous_region(
 250         unsigned long vstart, unsigned int order, unsigned int address_bits)
 251 {
 252         unsigned long *in_frames = discontig_frames, out_frame;
 253         unsigned long  frame, flags;
 254         unsigned int   i;
 255         int            rc, success;
 256         struct xen_memory_exchange exchange = {
 257                 .in = {
 258                         .nr_extents   = 1UL << order,
 259                         .extent_order = 0,
 260                         .domid        = DOMID_SELF
 261                 },
 262                 .out = {
 263                         .nr_extents   = 1,
 264                         .extent_order = order,
 265                         .address_bits = address_bits,
 266                         .domid        = DOMID_SELF
 267                 }
 268         };
 269
 270         /*
 271          * Currently an auto-translated guest will not perform I/O, nor will
 272          * it require PAE page directories below 4GB. Therefore any calls to
 273          * this function are redundant and can be ignored.
 274          */
 275         if (xen_feature(XENFEAT_auto_translated_physmap))
 276                 return 0;
 277
 278         if (unlikely(order > MAX_CONTIG_ORDER))
 279                 return -ENOMEM;
 280
 281         set_xen_guest_handle(exchange.in.extent_start, in_frames);
 282         set_xen_guest_handle(exchange.out.extent_start, &out_frame);
 283
 284         scrub_pages((void *)vstart, 1 << order);
 285
 286         balloon_lock(flags);
 287
 288         /* 1. Zap current PTEs, remembering MFNs. */
 289         for (i = 0; i < (1U<<order); i++) {
 290                 in_frames[i] = pfn_to_mfn((__pa(vstart) >> PAGE_SHIFT) + i);
 291                 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
 292                                         __pte_ma(0), 0);
 293                 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i,
 294                         INVALID_P2M_ENTRY);
 295         }
 296         if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
 297                 BUG();
 298
 299         /* 2. Get a new contiguous memory extent. */
 300         out_frame = __pa(vstart) >> PAGE_SHIFT;
 301         rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
 302         success = (exchange.nr_exchanged == (1UL << order));
 303         BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
 304         BUG_ON(success && (rc != 0));
 305 #if CONFIG_XEN_COMPAT <= 0x030002
 306         if (unlikely(rc == -ENOSYS)) {
 307                 /* Compatibility when XENMEM_exchange is unsupported. */
 308                 if (HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 309                                          &exchange.in) != (1UL << order))
 310                         BUG();
 311                 success = (HYPERVISOR_memory_op(XENMEM_populate_physmap,
 312                                                 &exchange.out) == 1);
 313                 if (!success) {
 314                         /* Couldn't get special memory: fall back to normal. */
 315                         for (i = 0; i < (1U<<order); i++)
 316                                 in_frames[i] = (__pa(vstart)>>PAGE_SHIFT) + i;
 317                         if (HYPERVISOR_memory_op(XENMEM_populate_physmap,
 318                                                  &exchange.in) != (1UL<<order))
 319                                 BUG();
 320                 }
 321         }
 322 #endif
 323
 324         /* 3. Map the new extent in place of old pages. */
 325         for (i = 0; i < (1U<<order); i++) {
 326                 frame = success ? (out_frame + i) : in_frames[i];
 327                 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
 328                                         pfn_pte_ma(frame, PAGE_KERNEL), 0);
 329                 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, frame);
 330         }
 331
 332         cr_mcl[i - 1].args[MULTI_UVMFLAGS_INDEX] = order
 333                                                    ? UVMF_TLB_FLUSH|UVMF_ALL
 334                                                    : UVMF_INVLPG|UVMF_ALL;
 335         if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
 336                 BUG();
 337
 338         if (success)
 339                 contiguous_bitmap_set(__pa(vstart) >> PAGE_SHIFT,
 340                                       1UL << order);
 341
 342         balloon_unlock(flags);
 343
 344         return success ? 0 : -ENOMEM;
 345 }
 346 EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
 347
 348 void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
 349 {
 350         unsigned long *out_frames = discontig_frames, in_frame;
 351         unsigned long  frame, flags;
 352         unsigned int   i;
 353         int            rc, success;
 354         struct xen_memory_exchange exchange = {
 355                 .in = {
 356                         .nr_extents   = 1,
 357                         .extent_order = order,
 358                         .domid        = DOMID_SELF
 359                 },
 360                 .out = {
 361                         .nr_extents   = 1UL << order,
 362                         .extent_order = 0,
 363                         .domid        = DOMID_SELF
 364                 }
 365         };
 366
 367         if (xen_feature(XENFEAT_auto_translated_physmap) ||
 368             !test_bit(__pa(vstart) >> PAGE_SHIFT, contiguous_bitmap))
 369                 return;
 370
 371         if (unlikely(order > MAX_CONTIG_ORDER))
 372                 return;
 373
 374         set_xen_guest_handle(exchange.in.extent_start, &in_frame);
 375         set_xen_guest_handle(exchange.out.extent_start, out_frames);
 376
 377         scrub_pages((void *)vstart, 1 << order);
 378
 379         balloon_lock(flags);
 380
 381         contiguous_bitmap_clear(__pa(vstart) >> PAGE_SHIFT, 1UL << order);
 382
 383         /* 1. Find start MFN of contiguous extent. */
 384         in_frame = pfn_to_mfn(__pa(vstart) >> PAGE_SHIFT);
 385
 386         /* 2. Zap current PTEs. */
 387         for (i = 0; i < (1U<<order); i++) {
 388                 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
 389                                         __pte_ma(0), 0);
 390                 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i,
 391                         INVALID_P2M_ENTRY);
 392                 out_frames[i] = (__pa(vstart) >> PAGE_SHIFT) + i;
 393         }
 394         if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
 395                 BUG();
 396
 397         /* 3. Do the exchange for non-contiguous MFNs. */
 398         rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
 399         success = (exchange.nr_exchanged == 1);
 400         BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
 401         BUG_ON(success && (rc != 0));
 402 #if CONFIG_XEN_COMPAT <= 0x030002
 403         if (unlikely(rc == -ENOSYS)) {
 404                 /* Compatibility when XENMEM_exchange is unsupported. */
 405                 if (HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 406                                          &exchange.in) != 1)
 407                         BUG();
 408                 if (HYPERVISOR_memory_op(XENMEM_populate_physmap,
 409                                          &exchange.out) != (1UL << order))
 410                         BUG();
 411                 success = 1;
 412         }
 413 #endif
 414
 415         /* 4. Map new pages in place of old pages. */
 416         for (i = 0; i < (1U<<order); i++) {
 417                 frame = success ? out_frames[i] : (in_frame + i);
 418                 MULTI_update_va_mapping(cr_mcl + i, vstart + (i*PAGE_SIZE),
 419                                         pfn_pte_ma(frame, PAGE_KERNEL), 0);
 420                 set_phys_to_machine((__pa(vstart)>>PAGE_SHIFT)+i, frame);
 421         }
 422
 423         cr_mcl[i - 1].args[MULTI_UVMFLAGS_INDEX] = order
 424                                                    ? UVMF_TLB_FLUSH|UVMF_ALL
 425                                                    : UVMF_INVLPG|UVMF_ALL;
 426         if (HYPERVISOR_multicall_check(cr_mcl, i, NULL))
 427                 BUG();
 428
 429         balloon_unlock(flags);
 430
 431         if (unlikely(!success)) {
 432                 /* Try hard to get the special memory back to Xen. */
 433                 exchange.in.extent_order = 0;
 434                 set_xen_guest_handle(exchange.in.extent_start, &in_frame);
 435
 436                 for (i = 0; i < (1U<<order); i++) {
 437                         struct page *page = alloc_page(__GFP_HIGHMEM|__GFP_COLD);
 438                         unsigned long pfn;
 439                         mmu_update_t mmu;
 440                         unsigned int j = 0;
 441
 442                         if (!page) {
 443                                 printk(KERN_WARNING "Xen and kernel out of memory "
 444                                        "while trying to release an order %u "
 445                                        "contiguous region\n", order);
 446                                 break;
 447                         }
 448                         pfn = page_to_pfn(page);
 449
 450                         balloon_lock(flags);
 451
 452                         if (!PageHighMem(page)) {
 453                                 void *v = __va(pfn << PAGE_SHIFT);
 454
 455                                 scrub_pages(v, 1);
 456                                 MULTI_update_va_mapping(cr_mcl + j, (unsigned long)v,
 457                                                         __pte_ma(0), UVMF_INVLPG|UVMF_ALL);
 458                                 ++j;
 459                         }
 460 #ifdef CONFIG_XEN_SCRUB_PAGES
 461                         else {
 462                                 scrub_pages(kmap(page), 1);
 463                                 kunmap(page);
 464                                 kmap_flush_unused();
 465                         }
 466 #endif
 467
 468                         frame = pfn_to_mfn(pfn);
 469                         set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 470
 471                         MULTI_update_va_mapping(cr_mcl + j, vstart,
 472                                                 pfn_pte_ma(frame, PAGE_KERNEL),
 473                                                 UVMF_INVLPG|UVMF_ALL);
 474                         ++j;
 475
 476                         pfn = __pa(vstart) >> PAGE_SHIFT;
 477                         set_phys_to_machine(pfn, frame);
 478                         if (!xen_feature(XENFEAT_auto_translated_physmap)) {
 479                                 mmu.ptr = ((uint64_t)frame << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
 480                                 mmu.val = pfn;
 481                                 cr_mcl[j].op = __HYPERVISOR_mmu_update;
 482                                 cr_mcl[j].args[0] = (unsigned long)&mmu;
 483                                 cr_mcl[j].args[1] = 1;
 484                                 cr_mcl[j].args[2] = 0;
 485                                 cr_mcl[j].args[3] = DOMID_SELF;
 486                                 ++j;
 487                         }
 488
 489                         cr_mcl[j].op = __HYPERVISOR_memory_op;
 490                         cr_mcl[j].args[0] = XENMEM_decrease_reservation;
 491                         cr_mcl[j].args[1] = (unsigned long)&exchange.in;
 492
 493                         if (HYPERVISOR_multicall(cr_mcl, j + 1))
 494                                 BUG();
 495                         BUG_ON(cr_mcl[j].result != 1);
 496                         while (j--)
 497                                 BUG_ON(cr_mcl[j].result != 0);
 498
 499                         balloon_unlock(flags);
 500
 501                         free_empty_pages(&page, 1);
 502
 503                         in_frame++;
 504                         vstart += PAGE_SIZE;
 505                 }
 506         }
 507 }
 508 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
 509
 510 int xen_limit_pages_to_max_mfn(
 511         struct page *pages, unsigned int order, unsigned int address_bits)
 512 {
 513         unsigned long flags, frame;
 514         unsigned long *in_frames = discontig_frames, *out_frames = limited_frames;
 515         struct page *page;
 516         unsigned int i, n, nr_mcl;
 517         int rc, success;
 518         DECLARE_BITMAP(limit_map, 1 << MAX_CONTIG_ORDER);
 519
 520         struct xen_memory_exchange exchange = {
 521                 .in = {
 522                         .extent_order = 0,
 523                         .domid        = DOMID_SELF
 524                 },
 525                 .out = {
 526                         .extent_order = 0,
 527                         .address_bits = address_bits,
 528                         .domid        = DOMID_SELF
 529                 }
 530         };
 531
 532         if (xen_feature(XENFEAT_auto_translated_physmap))
 533                 return 0;
 534
 535         if (unlikely(order > MAX_CONTIG_ORDER))
 536                 return -ENOMEM;
 537
 538         bitmap_zero(limit_map, 1U << order);
 539         set_xen_guest_handle(exchange.in.extent_start, in_frames);
 540         set_xen_guest_handle(exchange.out.extent_start, out_frames);
 541
 542         /* 0. Scrub the pages. */
 543         for (i = 0, n = 0; i < 1U<<order ; i++) {
 544                 page = &pages[i];
 545                 if (!(pfn_to_mfn(page_to_pfn(page)) >> (address_bits - PAGE_SHIFT)))
 546                         continue;
 547                 __set_bit(i, limit_map);
 548
 549                 if (!PageHighMem(page))
 550                         scrub_pages(page_address(page), 1);
 551 #ifdef CONFIG_XEN_SCRUB_PAGES
 552                 else {
 553                         scrub_pages(kmap(page), 1);
 554                         kunmap(page);
 555                         ++n;
 556                 }
 557 #endif
 558         }
 559         if (bitmap_empty(limit_map, 1U << order))
 560                 return 0;
 561
 562         if (n)
 563                 kmap_flush_unused();
 564
 565         balloon_lock(flags);
 566
 567         /* 1. Zap current PTEs (if any), remembering MFNs. */
 568         for (i = 0, n = 0, nr_mcl = 0; i < (1U<<order); i++) {
 569                 if(!test_bit(i, limit_map))
 570                         continue;
 571                 page = &pages[i];
 572
 573                 out_frames[n] = page_to_pfn(page);
 574                 in_frames[n] = pfn_to_mfn(out_frames[n]);
 575
 576                 if (!PageHighMem(page))
 577                         MULTI_update_va_mapping(cr_mcl + nr_mcl++,
 578                                                 (unsigned long)page_address(page),
 579                                                 __pte_ma(0), 0);
 580
 581                 set_phys_to_machine(out_frames[n], INVALID_P2M_ENTRY);
 582                 ++n;
 583         }
 584         if (nr_mcl && HYPERVISOR_multicall_check(cr_mcl, nr_mcl, NULL))
 585                 BUG();
 586
 587         /* 2. Get new memory below the required limit. */
 588         exchange.in.nr_extents = n;
 589         exchange.out.nr_extents = n;
 590         rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
 591         success = (exchange.nr_exchanged == n);
 592         BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
 593         BUG_ON(success && (rc != 0));
 594 #if CONFIG_XEN_COMPAT <= 0x030002
 595         if (unlikely(rc == -ENOSYS)) {
 596                 /* Compatibility when XENMEM_exchange is unsupported. */
 597                 if (HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 598                                          &exchange.in) != n)
 599                         BUG();
 600                 if (HYPERVISOR_memory_op(XENMEM_populate_physmap,
 601                                          &exchange.out) != n)
 602                         BUG();
 603                 success = 1;
 604         }
 605 #endif
 606
 607         /* 3. Map the new pages in place of old pages. */
 608         for (i = 0, n = 0, nr_mcl = 0; i < (1U<<order); i++) {
 609                 if(!test_bit(i, limit_map))
 610                         continue;
 611                 page = &pages[i];
 612
 613                 frame = success ? out_frames[n] : in_frames[n];
 614
 615                 if (!PageHighMem(page))
 616                         MULTI_update_va_mapping(cr_mcl + nr_mcl++,
 617                                                 (unsigned long)page_address(page),
 618                                                 pfn_pte_ma(frame, PAGE_KERNEL), 0);
 619
 620                 set_phys_to_machine(page_to_pfn(page), frame);
 621                 ++n;
 622         }
 623         if (nr_mcl) {
 624                 cr_mcl[nr_mcl - 1].args[MULTI_UVMFLAGS_INDEX] = order
 625                                                                 ? UVMF_TLB_FLUSH|UVMF_ALL
 626                                                                 : UVMF_INVLPG|UVMF_ALL;
 627                 if (HYPERVISOR_multicall_check(cr_mcl, nr_mcl, NULL))
 628                         BUG();
 629         }
 630
 631         balloon_unlock(flags);
 632
 633         return success ? 0 : -ENOMEM;
 634 }
 635 EXPORT_SYMBOL_GPL(xen_limit_pages_to_max_mfn);
 636
 637 #ifdef __i386__
 638 int write_ldt_entry(void *ldt, int entry, __u32 entry_a, __u32 entry_b)
 639 {
 640         __u32 *lp = (__u32 *)((char *)ldt + entry * 8);
 641         maddr_t mach_lp = arbitrary_virt_to_machine(lp);
 642         return HYPERVISOR_update_descriptor(
 643                 mach_lp, (u64)entry_a | ((u64)entry_b<<32));
 644 }
 645 #endif
 646
 647 #define MAX_BATCHED_FULL_PTES 32
 648
 649 int xen_change_pte_range(struct mm_struct *mm, pmd_t *pmd,
 650                          unsigned long addr, unsigned long end, pgprot_t newprot,
 651                          int dirty_accountable)
 652 {
 653         int rc = 0, i = 0;
 654         mmu_update_t u[MAX_BATCHED_FULL_PTES];
 655         pte_t *pte;
 656         spinlock_t *ptl;
 657
 658         if (!xen_feature(XENFEAT_mmu_pt_update_preserve_ad))
 659                 return 0;
 660
 661         pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
 662         do {
 663                 if (pte_present(*pte)) {
 664                         pte_t ptent = pte_modify(*pte, newprot);
 665
 666                         if (dirty_accountable && pte_dirty(ptent))
 667                                 ptent = pte_mkwrite(ptent);
 668                         u[i].ptr = (__pmd_val(*pmd) & PHYSICAL_PAGE_MASK)
 669                                    | ((unsigned long)pte & ~PAGE_MASK)
 670                                    | MMU_PT_UPDATE_PRESERVE_AD;
 671                         u[i].val = __pte_val(ptent);
 672                         if (++i == MAX_BATCHED_FULL_PTES) {
 673                                 if ((rc = HYPERVISOR_mmu_update(
 674                                         &u[0], i, NULL, DOMID_SELF)) != 0)
 675                                         break;
 676                                 i = 0;
 677                         }
 678                 }
 679         } while (pte++, addr += PAGE_SIZE, addr != end);
 680         if (i)
 681                 rc = HYPERVISOR_mmu_update( &u[0], i, NULL, DOMID_SELF);
 682         pte_unmap_unlock(pte - 1, ptl);
 683         BUG_ON(rc && rc != -ENOSYS);
 684         return !rc;
 685 }