drivers/xen/core/gnttab.c

   1 /******************************************************************************
   2  * gnttab.c
   3  *
   4  * Granting foreign access to our memory reservation.
   5  *
   6  * Copyright (c) 2005-2006, Christopher Clark
   7  * Copyright (c) 2004-2005, K A Fraser
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License version 2
  11  * as published by the Free Software Foundation; or, when distributed
  12  * separately from the Linux kernel or incorporated into other
  13  * software packages, subject to the following license:
  14  *
  15  * Permission is hereby granted, free of charge, to any person obtaining a copy
  16  * of this source file (the "Software"), to deal in the Software without
  17  * restriction, including without limitation the rights to use, copy, modify,
  18  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19  * and to permit persons to whom the Software is furnished to do so, subject to
  20  * the following conditions:
  21  *
  22  * The above copyright notice and this permission notice shall be included in
  23  * all copies or substantial portions of the Software.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31  * IN THE SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/mm.h>
  37 #include <xen/interface/xen.h>
  38 #include <xen/gnttab.h>
  39 #include <asm/pgtable.h>
  40 #include <asm/uaccess.h>
  41 #include <asm/synch_bitops.h>
  42 #include <asm/io.h>
  43 #include <xen/interface/memory.h>
  44 #include <xen/driver_util.h>
  45
  46 #ifdef HAVE_XEN_PLATFORM_COMPAT_H
  47 #include <xen/platform-compat.h>
  48 #endif
  49
  50 /* External tools reserve first few grant table entries. */
  51 #define NR_RESERVED_ENTRIES 8
  52 #define GNTTAB_LIST_END 0xffffffff
  53 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
  54
  55 static grant_ref_t **gnttab_list;
  56 static unsigned int nr_grant_frames;
  57 static unsigned int boot_max_nr_grant_frames;
  58 static int gnttab_free_count;
  59 static grant_ref_t gnttab_free_head;
  60 static DEFINE_SPINLOCK(gnttab_list_lock);
  61
  62 static struct grant_entry *shared;
  63
  64 static struct gnttab_free_callback *gnttab_free_callback_list;
  65
  66 static int gnttab_expand(unsigned int req_entries);
  67
  68 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
  69 #define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
  70
  71 static int get_free_entries(int count)
  72 {
  73         unsigned long flags;
  74         int ref, rc;
  75         grant_ref_t head;
  76
  77         spin_lock_irqsave(&gnttab_list_lock, flags);
  78
  79         if ((gnttab_free_count < count) &&
  80             ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
  81                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
  82                 return rc;
  83         }
  84
  85         ref = head = gnttab_free_head;
  86         gnttab_free_count -= count;
  87         while (count-- > 1)
  88                 head = gnttab_entry(head);
  89         gnttab_free_head = gnttab_entry(head);
  90         gnttab_entry(head) = GNTTAB_LIST_END;
  91
  92         spin_unlock_irqrestore(&gnttab_list_lock, flags);
  93
  94         return ref;
  95 }
  96
  97 #define get_free_entry() get_free_entries(1)
  98
  99 static void do_free_callbacks(void)
 100 {
 101         struct gnttab_free_callback *callback, *next;
 102
 103         callback = gnttab_free_callback_list;
 104         gnttab_free_callback_list = NULL;
 105
 106         while (callback != NULL) {
 107                 next = callback->next;
 108                 if (gnttab_free_count >= callback->count) {
 109                         callback->next = NULL;
 110                         callback->fn(callback->arg);
 111                 } else {
 112                         callback->next = gnttab_free_callback_list;
 113                         gnttab_free_callback_list = callback;
 114                 }
 115                 callback = next;
 116         }
 117 }
 118
 119 static inline void check_free_callbacks(void)
 120 {
 121         if (unlikely(gnttab_free_callback_list))
 122                 do_free_callbacks();
 123 }
 124
 125 static void put_free_entry(grant_ref_t ref)
 126 {
 127         unsigned long flags;
 128         spin_lock_irqsave(&gnttab_list_lock, flags);
 129         gnttab_entry(ref) = gnttab_free_head;
 130         gnttab_free_head = ref;
 131         gnttab_free_count++;
 132         check_free_callbacks();
 133         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 134 }
 135
 136 /*
 137  * Public grant-issuing interface functions
 138  */
 139
 140 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 141                                 int readonly)
 142 {
 143         int ref;
 144
 145         if (unlikely((ref = get_free_entry()) < 0))
 146                 return -ENOSPC;
 147
 148         shared[ref].frame = frame;
 149         shared[ref].domid = domid;
 150         wmb();
 151         shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
 152
 153         return ref;
 154 }
 155 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 156
 157 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 158                                      unsigned long frame, int readonly)
 159 {
 160         shared[ref].frame = frame;
 161         shared[ref].domid = domid;
 162         wmb();
 163         shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
 164 }
 165 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 166
 167
 168 int gnttab_query_foreign_access(grant_ref_t ref)
 169 {
 170         u16 nflags;
 171
 172         nflags = shared[ref].flags;
 173
 174         return (nflags & (GTF_reading|GTF_writing));
 175 }
 176 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 177
 178 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 179 {
 180         u16 flags, nflags;
 181
 182         nflags = shared[ref].flags;
 183         do {
 184                 if ((flags = nflags) & (GTF_reading|GTF_writing)) {
 185                         printk(KERN_ALERT "WARNING: g.e. still in use!\n");
 186                         return 0;
 187                 }
 188         } while ((nflags = synch_cmpxchg_subword(&shared[ref].flags, flags, 0)) !=
 189                  flags);
 190
 191         return 1;
 192 }
 193 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 194
 195 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 196                                unsigned long page)
 197 {
 198         if (gnttab_end_foreign_access_ref(ref, readonly)) {
 199                 put_free_entry(ref);
 200                 if (page != 0)
 201                         free_page(page);
 202         } else {
 203                 /* XXX This needs to be fixed so that the ref and page are
 204                    placed on a list to be freed up later. */
 205                 printk(KERN_WARNING
 206                        "WARNING: leaking g.e. and page still in use!\n");
 207         }
 208 }
 209 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 210
 211 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 212 {
 213         int ref;
 214
 215         if (unlikely((ref = get_free_entry()) < 0))
 216                 return -ENOSPC;
 217         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 218
 219         return ref;
 220 }
 221 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 222
 223 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 224                                        unsigned long pfn)
 225 {
 226         shared[ref].frame = pfn;
 227         shared[ref].domid = domid;
 228         wmb();
 229         shared[ref].flags = GTF_accept_transfer;
 230 }
 231 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 232
 233 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 234 {
 235         unsigned long frame;
 236         u16           flags;
 237
 238         /*
 239          * If a transfer is not even yet started, try to reclaim the grant
 240          * reference and return failure (== 0).
 241          */
 242         while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
 243                 if (synch_cmpxchg_subword(&shared[ref].flags, flags, 0) == flags)
 244                         return 0;
 245                 cpu_relax();
 246         }
 247
 248         /* If a transfer is in progress then wait until it is completed. */
 249         while (!(flags & GTF_transfer_completed)) {
 250                 flags = shared[ref].flags;
 251                 cpu_relax();
 252         }
 253
 254         /* Read the frame number /after/ reading completion status. */
 255         rmb();
 256         frame = shared[ref].frame;
 257         BUG_ON(frame == 0);
 258
 259         return frame;
 260 }
 261 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 262
 263 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 264 {
 265         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 266         put_free_entry(ref);
 267         return frame;
 268 }
 269 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 270
 271 void gnttab_free_grant_reference(grant_ref_t ref)
 272 {
 273         put_free_entry(ref);
 274 }
 275 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 276
 277 void gnttab_free_grant_references(grant_ref_t head)
 278 {
 279         grant_ref_t ref;
 280         unsigned long flags;
 281         int count = 1;
 282         if (head == GNTTAB_LIST_END)
 283                 return;
 284         spin_lock_irqsave(&gnttab_list_lock, flags);
 285         ref = head;
 286         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 287                 ref = gnttab_entry(ref);
 288                 count++;
 289         }
 290         gnttab_entry(ref) = gnttab_free_head;
 291         gnttab_free_head = head;
 292         gnttab_free_count += count;
 293         check_free_callbacks();
 294         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 295 }
 296 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 297
 298 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 299 {
 300         int h = get_free_entries(count);
 301
 302         if (h < 0)
 303                 return -ENOSPC;
 304
 305         *head = h;
 306
 307         return 0;
 308 }
 309 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 310
 311 int gnttab_empty_grant_references(const grant_ref_t *private_head)
 312 {
 313         return (*private_head == GNTTAB_LIST_END);
 314 }
 315 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 316
 317 int gnttab_claim_grant_reference(grant_ref_t *private_head)
 318 {
 319         grant_ref_t g = *private_head;
 320         if (unlikely(g == GNTTAB_LIST_END))
 321                 return -ENOSPC;
 322         *private_head = gnttab_entry(g);
 323         return g;
 324 }
 325 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 326
 327 void gnttab_release_grant_reference(grant_ref_t *private_head,
 328                                     grant_ref_t release)
 329 {
 330         gnttab_entry(release) = *private_head;
 331         *private_head = release;
 332 }
 333 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 334
 335 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 336                                   void (*fn)(void *), void *arg, u16 count)
 337 {
 338         unsigned long flags;
 339         spin_lock_irqsave(&gnttab_list_lock, flags);
 340         if (callback->next)
 341                 goto out;
 342         callback->fn = fn;
 343         callback->arg = arg;
 344         callback->count = count;
 345         callback->next = gnttab_free_callback_list;
 346         gnttab_free_callback_list = callback;
 347         check_free_callbacks();
 348 out:
 349         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 350 }
 351 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 352
 353 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 354 {
 355         struct gnttab_free_callback **pcb;
 356         unsigned long flags;
 357
 358         spin_lock_irqsave(&gnttab_list_lock, flags);
 359         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 360                 if (*pcb == callback) {
 361                         *pcb = callback->next;
 362                         break;
 363                 }
 364         }
 365         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 366 }
 367 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 368
 369 static int grow_gnttab_list(unsigned int more_frames)
 370 {
 371         unsigned int new_nr_grant_frames, extra_entries, i;
 372
 373         new_nr_grant_frames = nr_grant_frames + more_frames;
 374         extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
 375
 376         for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
 377         {
 378                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 379                 if (!gnttab_list[i])
 380                         goto grow_nomem;
 381         }
 382
 383
 384         for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 385              i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
 386                 gnttab_entry(i) = i + 1;
 387
 388         gnttab_entry(i) = gnttab_free_head;
 389         gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 390         gnttab_free_count += extra_entries;
 391
 392         nr_grant_frames = new_nr_grant_frames;
 393
 394         check_free_callbacks();
 395
 396         return 0;
 397
 398 grow_nomem:
 399         for ( ; i >= nr_grant_frames; i--)
 400                 free_page((unsigned long) gnttab_list[i]);
 401         return -ENOMEM;
 402 }
 403
 404 static unsigned int __max_nr_grant_frames(void)
 405 {
 406         struct gnttab_query_size query;
 407         int rc;
 408
 409         query.dom = DOMID_SELF;
 410
 411         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 412         if ((rc < 0) || (query.status != GNTST_okay))
 413                 return 4; /* Legacy max supported number of frames */
 414
 415         return query.max_nr_frames;
 416 }
 417
 418 static inline unsigned int max_nr_grant_frames(void)
 419 {
 420         unsigned int xen_max = __max_nr_grant_frames();
 421
 422         if (xen_max > boot_max_nr_grant_frames)
 423                 return boot_max_nr_grant_frames;
 424         return xen_max;
 425 }
 426
 427 #ifdef CONFIG_XEN
 428
 429 #ifndef __ia64__
 430 static int map_pte_fn(pte_t *pte, struct page *pmd_page,
 431                       unsigned long addr, void *data)
 432 {
 433         unsigned long **frames = (unsigned long **)data;
 434
 435         set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL));
 436         (*frames)++;
 437         return 0;
 438 }
 439
 440 static int unmap_pte_fn(pte_t *pte, struct page *pmd_page,
 441                         unsigned long addr, void *data)
 442 {
 443
 444         set_pte_at(&init_mm, addr, pte, __pte(0));
 445         return 0;
 446 }
 447 #endif
 448
 449 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 450 {
 451         struct gnttab_setup_table setup;
 452         unsigned long *frames;
 453         unsigned int nr_gframes = end_idx + 1;
 454         int rc;
 455
 456         frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
 457         if (!frames)
 458                 return -ENOMEM;
 459
 460         setup.dom        = DOMID_SELF;
 461         setup.nr_frames  = nr_gframes;
 462         set_xen_guest_handle(setup.frame_list, frames);
 463
 464         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
 465         if (rc == -ENOSYS) {
 466                 kfree(frames);
 467                 return -ENOSYS;
 468         }
 469
 470         BUG_ON(rc || setup.status);
 471
 472 #ifndef __ia64__
 473         if (shared == NULL) {
 474                 struct vm_struct *area;
 475                 area = alloc_vm_area(PAGE_SIZE * max_nr_grant_frames());
 476                 BUG_ON(area == NULL);
 477                 shared = area->addr;
 478         }
 479         rc = apply_to_page_range(&init_mm, (unsigned long)shared,
 480                                  PAGE_SIZE * nr_gframes,
 481                                  map_pte_fn, &frames);
 482         BUG_ON(rc);
 483         frames -= nr_gframes; /* adjust after map_pte_fn() */
 484 #else
 485         shared = __va(frames[0] << PAGE_SHIFT);
 486 #endif
 487
 488         kfree(frames);
 489
 490         return 0;
 491 }
 492
 493 int gnttab_resume(void)
 494 {
 495         if (max_nr_grant_frames() < nr_grant_frames)
 496                 return -ENOSYS;
 497         return gnttab_map(0, nr_grant_frames - 1);
 498 }
 499
 500 int gnttab_suspend(void)
 501 {
 502 #ifndef __ia64__
 503         apply_to_page_range(&init_mm, (unsigned long)shared,
 504                             PAGE_SIZE * nr_grant_frames,
 505                             unmap_pte_fn, NULL);
 506 #endif
 507         return 0;
 508 }
 509
 510 #else /* !CONFIG_XEN */
 511
 512 #include <platform-pci.h>
 513
 514 static unsigned long resume_frames;
 515
 516 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 517 {
 518         struct xen_add_to_physmap xatp;
 519         unsigned int i = end_idx;
 520
 521         /* Loop backwards, so that the first hypercall has the largest index,
 522          * ensuring that the table will grow only once.
 523          */
 524         do {
 525                 xatp.domid = DOMID_SELF;
 526                 xatp.idx = i;
 527                 xatp.space = XENMAPSPACE_grant_table;
 528                 xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
 529                 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
 530                         BUG();
 531         } while (i-- > start_idx);
 532
 533         return 0;
 534 }
 535
 536 int gnttab_resume(void)
 537 {
 538         unsigned int max_nr_gframes, nr_gframes;
 539
 540         nr_gframes = nr_grant_frames;
 541         max_nr_gframes = max_nr_grant_frames();
 542         if (max_nr_gframes < nr_gframes)
 543                 return -ENOSYS;
 544
 545         if (!resume_frames) {
 546                 resume_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
 547                 shared = ioremap(resume_frames, PAGE_SIZE * max_nr_gframes);
 548                 if (shared == NULL) {
 549                         printk("error to ioremap gnttab share frames\n");
 550                         return -1;
 551                 }
 552         }
 553
 554         gnttab_map(0, nr_gframes - 1);
 555
 556         return 0;
 557 }
 558
 559 #endif /* !CONFIG_XEN */
 560
 561 static int gnttab_expand(unsigned int req_entries)
 562 {
 563         int rc;
 564         unsigned int cur, extra;
 565
 566         cur = nr_grant_frames;
 567         extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
 568                  GREFS_PER_GRANT_FRAME);
 569         if (cur + extra > max_nr_grant_frames())
 570                 return -ENOSPC;
 571
 572         if ((rc = gnttab_map(cur, cur + extra - 1)) == 0)
 573                 rc = grow_gnttab_list(extra);
 574
 575         return rc;
 576 }
 577
 578 int __devinit gnttab_init(void)
 579 {
 580         int i;
 581         unsigned int max_nr_glist_frames;
 582         unsigned int nr_init_grefs;
 583
 584         if (!is_running_on_xen())
 585                 return -ENODEV;
 586
 587         nr_grant_frames = 1;
 588         boot_max_nr_grant_frames = __max_nr_grant_frames();
 589
 590         /* Determine the maximum number of frames required for the
 591          * grant reference free list on the current hypervisor.
 592          */
 593         max_nr_glist_frames = (boot_max_nr_grant_frames *
 594                                GREFS_PER_GRANT_FRAME /
 595                                (PAGE_SIZE / sizeof(grant_ref_t)));
 596
 597         gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
 598                               GFP_KERNEL);
 599         if (gnttab_list == NULL)
 600                 return -ENOMEM;
 601
 602         for (i = 0; i < nr_grant_frames; i++) {
 603                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
 604                 if (gnttab_list[i] == NULL)
 605                         goto ini_nomem;
 606         }
 607
 608         if (gnttab_resume() < 0)
 609                 return -ENODEV;
 610
 611         nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
 612
 613         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
 614                 gnttab_entry(i) = i + 1;
 615
 616         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
 617         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
 618         gnttab_free_head  = NR_RESERVED_ENTRIES;
 619
 620         return 0;
 621
 622  ini_nomem:
 623         for (i--; i >= 0; i--)
 624                 free_page((unsigned long)gnttab_list[i]);
 625         kfree(gnttab_list);
 626         return -ENOMEM;
 627 }
 628
 629 #ifdef CONFIG_XEN
 630 core_initcall(gnttab_init);
 631 #endif