drivers/xen/balloon/balloon.c

   1 /******************************************************************************
   2  * balloon.c
   3  *
   4  * Xen balloon driver - enables returning/claiming memory to/from Xen.
   5  *
   6  * Copyright (c) 2003, B Dragovic
   7  * Copyright (c) 2003-2004, M Williamson, K Fraser
   8  * Copyright (c) 2005 Dan M. Smith, IBM Corporation
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License version 2
  12  * as published by the Free Software Foundation; or, when distributed
  13  * separately from the Linux kernel or incorporated into other
  14  * software packages, subject to the following license:
  15  *
  16  * Permission is hereby granted, free of charge, to any person obtaining a copy
  17  * of this source file (the "Software"), to deal in the Software without
  18  * restriction, including without limitation the rights to use, copy, modify,
  19  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  20  * and to permit persons to whom the Software is furnished to do so, subject to
  21  * the following conditions:
  22  *
  23  * The above copyright notice and this permission notice shall be included in
  24  * all copies or substantial portions of the Software.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  27  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  28  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  29  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  30  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  31  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  32  * IN THE SOFTWARE.
  33  */
  34
  35 #include <linux/kernel.h>
  36 #include <linux/module.h>
  37 #include <linux/sched.h>
  38 #include <linux/errno.h>
  39 #include <linux/list.h>
  40 #include <linux/mm.h>
  41 #include <linux/swap.h>
  42 #include <linux/bootmem.h>
  43 #include <linux/highmem.h>
  44 #include <linux/slab.h>
  45 #include <linux/mutex.h>
  46 #include <xen/xen_proc.h>
  47 #include <asm/hypervisor.h>
  48 #include <xen/balloon.h>
  49 #include <xen/interface/memory.h>
  50 #include <asm/maddr.h>
  51 #include <asm/page.h>
  52 #include <asm/pgalloc.h>
  53 #include <asm/pgtable.h>
  54 #include <asm/uaccess.h>
  55 #include <asm/tlb.h>
  56 #include <xen/xenbus.h>
  57 #include "common.h"
  58
  59 #ifdef HAVE_XEN_PLATFORM_COMPAT_H
  60 #include <xen/platform-compat.h>
  61 #endif
  62
  63 #ifdef CONFIG_PROC_FS
  64 static struct proc_dir_entry *balloon_pde;
  65 #endif
  66
  67 static DEFINE_MUTEX(balloon_mutex);
  68
  69 /*
  70  * Protects atomic reservation decrease/increase against concurrent increases.
  71  * Also protects non-atomic updates of current_pages and driver_pages, and
  72  * balloon lists.
  73  */
  74 DEFINE_SPINLOCK(balloon_lock);
  75
  76 struct balloon_stats balloon_stats;
  77
  78 /* We increase/decrease in batches which fit in a page */
  79 static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)];
  80
  81 #ifdef CONFIG_HIGHMEM
  82 #define inc_totalhigh_pages() (totalhigh_pages++)
  83 #define dec_totalhigh_pages() (totalhigh_pages--)
  84 #else
  85 #define inc_totalhigh_pages() ((void)0)
  86 #define dec_totalhigh_pages() ((void)0)
  87 #endif
  88
  89 #ifndef CONFIG_XEN
  90 /*
  91  * In HVM guests accounting here uses the Xen visible values, but the kernel
  92  * determined totalram_pages value shouldn't get altered. Since totalram_pages
  93  * includes neither the kernel static image nor any memory allocated prior to
  94  * or from the bootmem allocator, we have to synchronize the two values.
  95  */
  96 static unsigned long __read_mostly totalram_bias;
  97 #else
  98 #define totalram_bias 0
  99 #endif
 100
 101 /* List of ballooned pages, threaded through the mem_map array. */
 102 static LIST_HEAD(ballooned_pages);
 103
 104 /* Main work function, always executed in process context. */
 105 static void balloon_process(struct work_struct *unused);
 106 static DECLARE_WORK(balloon_worker, balloon_process);
 107
 108 /* When ballooning out (allocating memory to return to Xen) we don't really
 109    want the kernel to try too hard since that can trigger the oom killer. */
 110 #define GFP_BALLOON (GFP_HIGHUSER|__GFP_NOWARN|__GFP_NORETRY|__GFP_NOMEMALLOC|\
 111                      __GFP_NOTRACK|__GFP_COLD)
 112
 113 #define PAGE_TO_LIST(p) (&(p)->lru)
 114 #define LIST_TO_PAGE(l) list_entry((l), struct page, lru)
 115 #define UNLIST_PAGE(p)                          \
 116         do {                                    \
 117                 list_del(PAGE_TO_LIST(p));      \
 118                 PAGE_TO_LIST(p)->next = NULL;   \
 119                 PAGE_TO_LIST(p)->prev = NULL;   \
 120         } while(0)
 121
 122 #define IPRINTK(fmt, args...) pr_info("xen_mem: " fmt, ##args)
 123 #define WPRINTK(fmt, args...) pr_warning("xen_mem: " fmt, ##args)
 124
 125 /* balloon_append: add the given page to the balloon. */
 126 static void balloon_append(struct page *page, int account)
 127 {
 128         unsigned long pfn;
 129
 130         /* Lowmem is re-populated first, so highmem pages go at list tail. */
 131         if (PageHighMem(page)) {
 132                 list_add_tail(PAGE_TO_LIST(page), &ballooned_pages);
 133                 bs.balloon_high++;
 134                 if (account)
 135                         dec_totalhigh_pages();
 136         } else {
 137                 list_add(PAGE_TO_LIST(page), &ballooned_pages);
 138                 bs.balloon_low++;
 139         }
 140
 141         pfn = page_to_pfn(page);
 142         if (account) {
 143                 SetPageReserved(page);
 144                 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 145                 page_zone(page)->present_pages--;
 146         } else {
 147                 BUG_ON(!PageReserved(page));
 148                 WARN_ON_ONCE(phys_to_machine_mapping_valid(pfn));
 149         }
 150 }
 151
 152 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
 153 static struct page *balloon_retrieve(int *was_empty)
 154 {
 155         struct page *page;
 156         struct zone *zone;
 157
 158         if (list_empty(&ballooned_pages))
 159                 return NULL;
 160
 161         page = LIST_TO_PAGE(ballooned_pages.next);
 162         UNLIST_PAGE(page);
 163         BUG_ON(!PageReserved(page));
 164
 165         if (PageHighMem(page)) {
 166                 bs.balloon_high--;
 167                 inc_totalhigh_pages();
 168         }
 169         else
 170                 bs.balloon_low--;
 171         zone = page_zone(page);
 172         *was_empty |= !populated_zone(zone);
 173         zone->present_pages++;
 174
 175         return page;
 176 }
 177
 178 static struct page *balloon_first_page(void)
 179 {
 180         if (list_empty(&ballooned_pages))
 181                 return NULL;
 182         return LIST_TO_PAGE(ballooned_pages.next);
 183 }
 184
 185 static struct page *balloon_next_page(struct page *page)
 186 {
 187         struct list_head *next = PAGE_TO_LIST(page)->next;
 188         if (next == &ballooned_pages)
 189                 return NULL;
 190         return LIST_TO_PAGE(next);
 191 }
 192
 193 static inline void balloon_free_page(struct page *page)
 194 {
 195 #ifndef MODULE
 196         if (put_page_testzero(page))
 197                 free_hot_cold_page(page, 1);
 198 #else
 199         /* free_hot_cold_page() is not being exported. */
 200         __free_page(page);
 201 #endif
 202 }
 203
 204 static void balloon_alarm(unsigned long unused)
 205 {
 206         schedule_work(&balloon_worker);
 207 }
 208 static DEFINE_TIMER(balloon_timer, balloon_alarm, 0, 0);
 209
 210 static unsigned long current_target(void)
 211 {
 212         unsigned long target = bs.target_pages;
 213         if (target > (bs.current_pages + bs.balloon_low + bs.balloon_high))
 214                 target = bs.current_pages + bs.balloon_low + bs.balloon_high;
 215         return target;
 216 }
 217
 218 unsigned long balloon_minimum_target(void)
 219 {
 220 #ifndef CONFIG_XEN
 221 #define max_pfn num_physpages
 222 #endif
 223         unsigned long min_pages, curr_pages = current_target();
 224
 225 #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
 226         /* Simple continuous piecewiese linear function:
 227          *  max MiB -> min MiB  gradient
 228          *       0         0
 229          *      16        16
 230          *      32        24
 231          *     128        72    (1/2)
 232          *     512       168    (1/4)
 233          *    2048       360    (1/8)
 234          *    8192       552    (1/32)
 235          *   32768      1320
 236          *  131072      4392
 237          */
 238         if (max_pfn < MB2PAGES(128))
 239                 min_pages = MB2PAGES(8) + (max_pfn >> 1);
 240         else if (max_pfn < MB2PAGES(512))
 241                 min_pages = MB2PAGES(40) + (max_pfn >> 2);
 242         else if (max_pfn < MB2PAGES(2048))
 243                 min_pages = MB2PAGES(104) + (max_pfn >> 3);
 244         else
 245                 min_pages = MB2PAGES(296) + (max_pfn >> 5);
 246 #undef MB2PAGES
 247
 248         /* Don't enforce growth */
 249         return min(min_pages, curr_pages);
 250 #ifndef CONFIG_XEN
 251 #undef max_pfn
 252 #endif
 253 }
 254
 255 static int increase_reservation(unsigned long nr_pages)
 256 {
 257         unsigned long  pfn, i, flags;
 258         struct page   *page;
 259         long           rc;
 260         int            need_zonelists_rebuild = 0;
 261         struct xen_memory_reservation reservation = {
 262                 .address_bits = 0,
 263                 .extent_order = 0,
 264                 .domid        = DOMID_SELF
 265         };
 266
 267         if (nr_pages > ARRAY_SIZE(frame_list))
 268                 nr_pages = ARRAY_SIZE(frame_list);
 269
 270         balloon_lock(flags);
 271
 272         page = balloon_first_page();
 273         for (i = 0; i < nr_pages; i++) {
 274                 BUG_ON(page == NULL);
 275                 frame_list[i] = page_to_pfn(page);;
 276                 page = balloon_next_page(page);
 277         }
 278
 279         set_xen_guest_handle(reservation.extent_start, frame_list);
 280         reservation.nr_extents = nr_pages;
 281         rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
 282         if (rc < 0)
 283                 goto out;
 284
 285         for (i = 0; i < rc; i++) {
 286                 page = balloon_retrieve(&need_zonelists_rebuild);
 287                 BUG_ON(page == NULL);
 288
 289                 pfn = page_to_pfn(page);
 290                 BUG_ON(!xen_feature(XENFEAT_auto_translated_physmap) &&
 291                        phys_to_machine_mapping_valid(pfn));
 292
 293                 set_phys_to_machine(pfn, frame_list[i]);
 294
 295 #ifdef CONFIG_XEN
 296                 /* Link back into the page tables if not highmem. */
 297                 if (pfn < max_low_pfn) {
 298                         int ret;
 299                         ret = HYPERVISOR_update_va_mapping(
 300                                 (unsigned long)__va(pfn << PAGE_SHIFT),
 301                                 pfn_pte_ma(frame_list[i], PAGE_KERNEL),
 302                                 0);
 303                         BUG_ON(ret);
 304                 }
 305 #endif
 306
 307                 /* Relinquish the page back to the allocator. */
 308                 ClearPageReserved(page);
 309                 init_page_count(page);
 310                 balloon_free_page(page);
 311         }
 312
 313         bs.current_pages += rc;
 314         totalram_pages = bs.current_pages - totalram_bias;
 315
 316  out:
 317         balloon_unlock(flags);
 318
 319 #ifndef MODULE
 320         setup_per_zone_wmarks();
 321         if (rc > 0)
 322                 kswapd_run(0);
 323         if (need_zonelists_rebuild)
 324                 build_all_zonelists(NULL);
 325         else
 326                 vm_total_pages = nr_free_pagecache_pages();
 327 #endif
 328
 329         return rc < 0 ? rc : rc != nr_pages;
 330 }
 331
 332 static int decrease_reservation(unsigned long nr_pages)
 333 {
 334         unsigned long  pfn, i, flags;
 335         struct page   *page;
 336         void          *v;
 337         int            need_sleep = 0;
 338         int ret;
 339         struct xen_memory_reservation reservation = {
 340                 .address_bits = 0,
 341                 .extent_order = 0,
 342                 .domid        = DOMID_SELF
 343         };
 344
 345         if (nr_pages > ARRAY_SIZE(frame_list))
 346                 nr_pages = ARRAY_SIZE(frame_list);
 347
 348         for (i = 0; i < nr_pages; i++) {
 349                 if ((page = alloc_page(GFP_BALLOON)) == NULL) {
 350                         nr_pages = i;
 351                         need_sleep = 1;
 352                         break;
 353                 }
 354
 355                 pfn = page_to_pfn(page);
 356                 frame_list[i] = pfn_to_mfn(pfn);
 357
 358                 if (!PageHighMem(page)) {
 359                         v = phys_to_virt(pfn << PAGE_SHIFT);
 360                         xen_scrub_pages(v, 1);
 361 #ifdef CONFIG_XEN
 362                         ret = HYPERVISOR_update_va_mapping(
 363                                 (unsigned long)v, __pte_ma(0), 0);
 364                         BUG_ON(ret);
 365 #endif
 366                 }
 367 #ifdef CONFIG_XEN_SCRUB_PAGES
 368                 else {
 369                         v = kmap(page);
 370                         xen_scrub_pages(v, 1);
 371                         kunmap(page);
 372                 }
 373 #endif
 374         }
 375
 376 #ifdef CONFIG_XEN
 377         /* Ensure that ballooned highmem pages don't have kmaps. */
 378         kmap_flush_unused();
 379         flush_tlb_all();
 380 #endif
 381
 382         balloon_lock(flags);
 383
 384         /* No more mappings: invalidate P2M and add to balloon. */
 385         for (i = 0; i < nr_pages; i++) {
 386                 pfn = mfn_to_pfn(frame_list[i]);
 387                 balloon_append(pfn_to_page(pfn), 1);
 388         }
 389
 390         set_xen_guest_handle(reservation.extent_start, frame_list);
 391         reservation.nr_extents   = nr_pages;
 392         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 393         BUG_ON(ret != nr_pages);
 394
 395         bs.current_pages -= nr_pages;
 396         totalram_pages = bs.current_pages - totalram_bias;
 397
 398         balloon_unlock(flags);
 399
 400         return need_sleep;
 401 }
 402
 403 /*
 404  * We avoid multiple worker processes conflicting via the balloon mutex.
 405  * We may of course race updates of the target counts (which are protected
 406  * by the balloon lock), or with changes to the Xen hard limit, but we will
 407  * recover from these in time.
 408  */
 409 static void balloon_process(struct work_struct *unused)
 410 {
 411         int need_sleep = 0;
 412         long credit;
 413
 414         mutex_lock(&balloon_mutex);
 415
 416         do {
 417                 credit = current_target() - bs.current_pages;
 418                 if (credit > 0)
 419                         need_sleep = (increase_reservation(credit) != 0);
 420                 if (credit < 0)
 421                         need_sleep = (decrease_reservation(-credit) != 0);
 422
 423 #ifndef CONFIG_PREEMPT
 424                 if (need_resched())
 425                         schedule();
 426 #endif
 427         } while ((credit != 0) && !need_sleep);
 428
 429         /* Schedule more work if there is some still to be done. */
 430         if (current_target() != bs.current_pages)
 431                 mod_timer(&balloon_timer, jiffies + HZ);
 432
 433         mutex_unlock(&balloon_mutex);
 434 }
 435
 436 /* Resets the Xen limit, sets new target, and kicks off processing. */
 437 void balloon_set_new_target(unsigned long target)
 438 {
 439         /* No need for lock. Not read-modify-write updates. */
 440         bs.target_pages = max(target, balloon_minimum_target());
 441         schedule_work(&balloon_worker);
 442 }
 443
 444 static struct xenbus_watch target_watch =
 445 {
 446         .node = "memory/target"
 447 };
 448
 449 /* React to a change in the target key */
 450 static void watch_target(struct xenbus_watch *watch,
 451                          const char **vec, unsigned int len)
 452 {
 453         unsigned long long new_target;
 454         int err;
 455
 456         err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
 457         if (err != 1) {
 458                 /* This is ok (for domain0 at least) - so just return */
 459                 return;
 460         }
 461
 462         /* The given memory/target value is in KiB, so it needs converting to
 463          * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 464          */
 465         balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
 466 }
 467
 468 static int balloon_init_watcher(struct notifier_block *notifier,
 469                                 unsigned long event,
 470                                 void *data)
 471 {
 472         int err;
 473
 474         err = register_xenbus_watch(&target_watch);
 475         if (err)
 476                 pr_err("Failed to set balloon watcher\n");
 477
 478         return NOTIFY_DONE;
 479 }
 480
 481 #ifdef CONFIG_PROC_FS
 482 static int balloon_write(struct file *file, const char __user *buffer,
 483                          unsigned long count, void *data)
 484 {
 485         char memstring[64], *endchar;
 486         unsigned long long target_bytes;
 487
 488         if (!capable(CAP_SYS_ADMIN))
 489                 return -EPERM;
 490
 491         if (count <= 1)
 492                 return -EBADMSG; /* runt */
 493         if (count > sizeof(memstring))
 494                 return -EFBIG;   /* too long */
 495
 496         if (copy_from_user(memstring, buffer, count))
 497                 return -EFAULT;
 498         memstring[sizeof(memstring)-1] = '\0';
 499
 500         target_bytes = memparse(memstring, &endchar);
 501         balloon_set_new_target(target_bytes >> PAGE_SHIFT);
 502
 503         return count;
 504 }
 505
 506 static int balloon_read(char *page, char **start, off_t off,
 507                         int count, int *eof, void *data)
 508 {
 509         int len;
 510
 511         len = sprintf(
 512                 page,
 513                 "Current allocation: %8lu kB\n"
 514                 "Requested target:   %8lu kB\n"
 515                 "Minimum target:     %8lu kB\n"
 516                 "Maximum target:     %8lu kB\n"
 517                 "Low-mem balloon:    %8lu kB\n"
 518                 "High-mem balloon:   %8lu kB\n"
 519                 "Driver pages:       %8lu kB\n",
 520                 PAGES2KB(bs.current_pages), PAGES2KB(bs.target_pages),
 521                 PAGES2KB(balloon_minimum_target()), PAGES2KB(num_physpages),
 522                 PAGES2KB(bs.balloon_low), PAGES2KB(bs.balloon_high),
 523                 PAGES2KB(bs.driver_pages));
 524
 525
 526         *eof = 1;
 527         return len;
 528 }
 529 #endif
 530
 531 static struct notifier_block xenstore_notifier;
 532
 533 static int __init balloon_init(void)
 534 {
 535 #if !defined(CONFIG_XEN)
 536 # ifndef XENMEM_get_pod_target
 537 #  define XENMEM_get_pod_target 17
 538         typedef struct xen_pod_target {
 539                 uint64_t target_pages;
 540                 uint64_t tot_pages;
 541                 uint64_t pod_cache_pages;
 542                 uint64_t pod_entries;
 543                 domid_t domid;
 544         } xen_pod_target_t;
 545 # endif
 546         xen_pod_target_t pod_target = { .domid = DOMID_SELF };
 547         int rc;
 548 #elif defined(CONFIG_X86)
 549         unsigned long pfn;
 550         struct page *page;
 551 #endif
 552
 553         if (!is_running_on_xen())
 554                 return -ENODEV;
 555
 556         IPRINTK("Initialising balloon driver.\n");
 557
 558 #ifdef CONFIG_XEN
 559         bs.current_pages = min(xen_start_info->nr_pages, max_pfn);
 560         totalram_pages   = bs.current_pages;
 561 #else
 562         rc = HYPERVISOR_memory_op(XENMEM_get_pod_target, &pod_target);
 563         /*
 564          * Xen prior to 3.4.0 masks the memory_op command to 4 bits, thus
 565          * converting XENMEM_get_pod_target to XENMEM_decrease_reservation.
 566          * Fortunately this results in a request with all input fields zero,
 567          * but (due to the way bit 4 and upwards get interpreted) a starting
 568          * extent of 1. When start_extent > nr_extents (>= in newer Xen), we
 569          * simply get start_extent returned.
 570          */
 571         totalram_bias = HYPERVISOR_memory_op(rc != -ENOSYS && rc != 1
 572                 ? XENMEM_maximum_reservation : XENMEM_current_reservation,
 573                 &pod_target.domid);
 574         if ((long)totalram_bias != -ENOSYS) {
 575                 BUG_ON(totalram_bias < totalram_pages);
 576                 bs.current_pages = totalram_bias;
 577                 totalram_bias -= totalram_pages;
 578         } else {
 579                 totalram_bias = 0;
 580                 bs.current_pages = totalram_pages;
 581         }
 582 #endif
 583         bs.target_pages  = bs.current_pages;
 584         bs.balloon_low   = 0;
 585         bs.balloon_high  = 0;
 586         bs.driver_pages  = 0UL;
 587
 588 #ifdef CONFIG_PROC_FS
 589         if ((balloon_pde = create_xen_proc_entry("balloon", 0644)) == NULL) {
 590                 WPRINTK("Unable to create /proc/xen/balloon.\n");
 591                 return -1;
 592         }
 593
 594         balloon_pde->read_proc  = balloon_read;
 595         balloon_pde->write_proc = balloon_write;
 596 #endif
 597         balloon_sysfs_init();
 598
 599 #if defined(CONFIG_X86) && defined(CONFIG_XEN)
 600         /* Initialise the balloon with excess memory space. */
 601         for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) {
 602                 page = pfn_to_page(pfn);
 603                 if (!PageReserved(page)) {
 604                         SetPageReserved(page);
 605                         set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 606                         balloon_append(page, 0);
 607                 }
 608         }
 609 #endif
 610
 611         target_watch.callback = watch_target;
 612         xenstore_notifier.notifier_call = balloon_init_watcher;
 613
 614         register_xenstore_notifier(&xenstore_notifier);
 615
 616         return 0;
 617 }
 618
 619 subsys_initcall(balloon_init);
 620
 621 static void __exit balloon_exit(void)
 622 {
 623         balloon_sysfs_exit();
 624         /* XXX - release balloon here */
 625 }
 626
 627 module_exit(balloon_exit);
 628
 629 void balloon_update_driver_allowance(long delta)
 630 {
 631         unsigned long flags;
 632
 633         balloon_lock(flags);
 634         bs.driver_pages += delta;
 635         balloon_unlock(flags);
 636 }
 637 EXPORT_SYMBOL_GPL(balloon_update_driver_allowance);
 638
 639 #if defined(CONFIG_XEN_BACKEND) || defined(CONFIG_XEN_BACKEND_MODULE)
 640
 641 #ifdef CONFIG_XEN
 642 static int dealloc_pte_fn(
 643         pte_t *pte, struct page *pmd_page, unsigned long addr, void *data)
 644 {
 645         unsigned long pfn, mfn = pte_mfn(*pte);
 646         int ret;
 647         struct xen_memory_reservation reservation = {
 648                 .nr_extents   = 1,
 649                 .extent_order = 0,
 650                 .domid        = DOMID_SELF
 651         };
 652         set_xen_guest_handle(reservation.extent_start, &mfn);
 653         set_pte_at(&init_mm, addr, pte, __pte_ma(0));
 654         pfn = __pa(addr) >> PAGE_SHIFT;
 655         set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
 656         SetPageReserved(pfn_to_page(pfn));
 657         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
 658         BUG_ON(ret != 1);
 659         return 0;
 660 }
 661 #endif
 662
 663 struct page **alloc_empty_pages_and_pagevec(int nr_pages)
 664 {
 665         unsigned long flags;
 666         void *v;
 667         struct page *page, **pagevec;
 668         int i, ret;
 669
 670         pagevec = kmalloc(sizeof(page) * nr_pages, GFP_KERNEL);
 671         if (pagevec == NULL)
 672                 return NULL;
 673
 674         for (i = 0; i < nr_pages; i++) {
 675                 balloon_lock(flags);
 676                 page = balloon_first_page();
 677                 if (page && !PageHighMem(page)) {
 678                         UNLIST_PAGE(page);
 679                         bs.balloon_low--;
 680                         balloon_unlock(flags);
 681                         pagevec[i] = page;
 682                         continue;
 683                 }
 684                 balloon_unlock(flags);
 685
 686                 page = pagevec[i] = alloc_page(GFP_KERNEL|__GFP_NOTRACK|__GFP_COLD);
 687                 if (page == NULL)
 688                         goto err;
 689
 690                 v = page_address(page);
 691                 xen_scrub_pages(v, 1);
 692
 693                 balloon_lock(flags);
 694
 695                 if (xen_feature(XENFEAT_auto_translated_physmap)) {
 696                         unsigned long gmfn = page_to_pfn(page);
 697                         struct xen_memory_reservation reservation = {
 698                                 .nr_extents   = 1,
 699                                 .extent_order = 0,
 700                                 .domid        = DOMID_SELF
 701                         };
 702                         set_xen_guest_handle(reservation.extent_start, &gmfn);
 703                         ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 704                                                    &reservation);
 705                         if (ret == 1)
 706                                 ret = 0; /* success */
 707                 } else {
 708 #ifdef CONFIG_XEN
 709                         ret = apply_to_page_range(&init_mm, (unsigned long)v,
 710                                                   PAGE_SIZE, dealloc_pte_fn,
 711                                                   NULL);
 712 #else
 713                         /* Cannot handle non-auto translate mode. */
 714                         ret = 1;
 715 #endif
 716                 }
 717
 718                 if (ret != 0) {
 719                         balloon_free_page(page);
 720                         balloon_unlock(flags);
 721                         goto err;
 722                 }
 723
 724                 totalram_pages = --bs.current_pages - totalram_bias;
 725                 if (PageHighMem(page))
 726                         dec_totalhigh_pages();
 727                 page_zone(page)->present_pages--;
 728
 729                 balloon_unlock(flags);
 730         }
 731
 732  out:
 733         schedule_work(&balloon_worker);
 734 #ifdef CONFIG_XEN
 735         flush_tlb_all();
 736 #endif
 737         return pagevec;
 738
 739  err:
 740         balloon_lock(flags);
 741         while (--i >= 0)
 742                 balloon_append(pagevec[i], 0);
 743         balloon_unlock(flags);
 744         kfree(pagevec);
 745         pagevec = NULL;
 746         goto out;
 747 }
 748 EXPORT_SYMBOL_GPL(alloc_empty_pages_and_pagevec);
 749
 750 #endif /* CONFIG_XEN_BACKEND */
 751
 752 #ifdef CONFIG_XEN
 753 static void _free_empty_pages(struct page **pagevec, int nr_pages,
 754                               bool account)
 755 {
 756         unsigned long flags;
 757         int i;
 758
 759         balloon_lock(flags);
 760         for (i = 0; i < nr_pages; i++) {
 761                 BUG_ON(page_count(pagevec[i]) != 1);
 762                 balloon_append(pagevec[i], account);
 763         }
 764         if (account) {
 765                 bs.current_pages -= nr_pages;
 766                 totalram_pages = bs.current_pages - totalram_bias;
 767         }
 768         balloon_unlock(flags);
 769
 770         schedule_work(&balloon_worker);
 771 }
 772
 773 void free_empty_pages(struct page **pagevec, int nr_pages)
 774 {
 775         _free_empty_pages(pagevec, nr_pages, true);
 776 }
 777 #endif
 778
 779 #if defined(CONFIG_XEN_BACKEND) || defined(CONFIG_XEN_BACKEND_MODULE)
 780 void free_empty_pages_and_pagevec(struct page **pagevec, int nr_pages)
 781 {
 782         if (pagevec) {
 783                 _free_empty_pages(pagevec, nr_pages, false);
 784                 kfree(pagevec);
 785         }
 786 }
 787 EXPORT_SYMBOL_GPL(free_empty_pages_and_pagevec);
 788 #endif
 789
 790 void balloon_release_driver_page(struct page *page)
 791 {
 792         unsigned long flags;
 793
 794         balloon_lock(flags);
 795         balloon_append(page, 1);
 796         totalram_pages = --bs.current_pages - totalram_bias;
 797         bs.driver_pages--;
 798         balloon_unlock(flags);
 799
 800         schedule_work(&balloon_worker);
 801 }
 802 EXPORT_SYMBOL_GPL(balloon_release_driver_page);
 803
 804 MODULE_LICENSE("Dual BSD/GPL");