drivers/xen/grant-table.c

   1 /******************************************************************************
   2  * grant_table.c
   3  *
   4  * Granting foreign access to our memory reservation.
   5  *
   6  * Copyright (c) 2005-2006, Christopher Clark
   7  * Copyright (c) 2004-2005, K A Fraser
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License version 2
  11  * as published by the Free Software Foundation; or, when distributed
  12  * separately from the Linux kernel or incorporated into other
  13  * software packages, subject to the following license:
  14  *
  15  * Permission is hereby granted, free of charge, to any person obtaining a copy
  16  * of this source file (the "Software"), to deal in the Software without
  17  * restriction, including without limitation the rights to use, copy, modify,
  18  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19  * and to permit persons to whom the Software is furnished to do so, subject to
  20  * the following conditions:
  21  *
  22  * The above copyright notice and this permission notice shall be included in
  23  * all copies or substantial portions of the Software.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31  * IN THE SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/mm.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/io.h>
  41
  42 #include <xen/xen.h>
  43 #include <xen/interface/xen.h>
  44 #include <xen/page.h>
  45 #include <xen/grant_table.h>
  46 #include <xen/interface/memory.h>
  47 #include <xen/hvc-console.h>
  48 #include <asm/xen/hypercall.h>
  49
  50 #include <asm/pgtable.h>
  51 #include <asm/sync_bitops.h>
  52
  53 /* External tools reserve first few grant table entries. */
  54 #define NR_RESERVED_ENTRIES 8
  55 #define GNTTAB_LIST_END 0xffffffff
  56 #define GREFS_PER_GRANT_FRAME \
  57 (grant_table_version == 1 ?                      \
  58 (PAGE_SIZE / sizeof(struct grant_entry_v1)) :   \
  59 (PAGE_SIZE / sizeof(union grant_entry_v2)))
  60
  61 static grant_ref_t **gnttab_list;
  62 static unsigned int nr_grant_frames;
  63 static unsigned int boot_max_nr_grant_frames;
  64 static int gnttab_free_count;
  65 static grant_ref_t gnttab_free_head;
  66 static DEFINE_SPINLOCK(gnttab_list_lock);
  67 unsigned long xen_hvm_resume_frames;
  68 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
  69
  70 static union {
  71         struct grant_entry_v1 *v1;
  72         union grant_entry_v2 *v2;
  73         void *addr;
  74 } gnttab_shared;
  75
  76 /*This is a structure of function pointers for grant table*/
  77 struct gnttab_ops {
  78         /*
  79          * Mapping a list of frames for storing grant entries. Frames parameter
  80          * is used to store grant table address when grant table being setup,
  81          * nr_gframes is the number of frames to map grant table. Returning
  82          * GNTST_okay means success and negative value means failure.
  83          */
  84         int (*map_frames)(unsigned long *frames, unsigned int nr_gframes);
  85         /*
  86          * Release a list of frames which are mapped in map_frames for grant
  87          * entry status.
  88          */
  89         void (*unmap_frames)(void);
  90         /*
  91          * Introducing a valid entry into the grant table, granting the frame of
  92          * this grant entry to domain for accessing or transfering. Ref
  93          * parameter is reference of this introduced grant entry, domid is id of
  94          * granted domain, frame is the page frame to be granted, and flags is
  95          * status of the grant entry to be updated.
  96          */
  97         void (*update_entry)(grant_ref_t ref, domid_t domid,
  98                              unsigned long frame, unsigned flags);
  99         /*
 100          * Stop granting a grant entry to domain for accessing. Ref parameter is
 101          * reference of a grant entry whose grant access will be stopped,
 102          * readonly is not in use in this function. If the grant entry is
 103          * currently mapped for reading or writing, just return failure(==0)
 104          * directly and don't tear down the grant access. Otherwise, stop grant
 105          * access for this entry and return success(==1).
 106          */
 107         int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
 108         /*
 109          * Stop granting a grant entry to domain for transfer. Ref parameter is
 110          * reference of a grant entry whose grant transfer will be stopped. If
 111          * tranfer has not started, just reclaim the grant entry and return
 112          * failure(==0). Otherwise, wait for the transfer to complete and then
 113          * return the frame.
 114          */
 115         unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
 116         /*
 117          * Query the status of a grant entry. Ref parameter is reference of
 118          * queried grant entry, return value is the status of queried entry.
 119          * Detailed status(writing/reading) can be gotten from the return value
 120          * by bit operations.
 121          */
 122         int (*query_foreign_access)(grant_ref_t ref);
 123         /*
 124          * Grant a domain to access a range of bytes within the page referred by
 125          * an available grant entry. Ref parameter is reference of a grant entry
 126          * which will be sub-page accessed, domid is id of grantee domain, frame
 127          * is frame address of subpage grant, flags is grant type and flag
 128          * information, page_off is offset of the range of bytes, and length is
 129          * length of bytes to be accessed.
 130          */
 131         void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
 132                                      unsigned long frame, int flags,
 133                                      unsigned page_off, unsigned length);
 134         /*
 135          * Redirect an available grant entry on domain A to another grant
 136          * reference of domain B, then allow domain C to use grant reference
 137          * of domain B transitively. Ref parameter is an available grant entry
 138          * reference on domain A, domid is id of domain C which accesses grant
 139          * entry transitively, flags is grant type and flag information,
 140          * trans_domid is id of domain B whose grant entry is finally accessed
 141          * transitively, trans_gref is grant entry transitive reference of
 142          * domain B.
 143          */
 144         void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
 145                                    domid_t trans_domid, grant_ref_t trans_gref);
 146 };
 147
 148 static struct gnttab_ops *gnttab_interface;
 149
 150 /*This reflects status of grant entries, so act as a global value*/
 151 static grant_status_t *grstatus;
 152
 153 static int grant_table_version;
 154
 155 static struct gnttab_free_callback *gnttab_free_callback_list;
 156
 157 static int gnttab_expand(unsigned int req_entries);
 158
 159 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 160 #define SPP (PAGE_SIZE / sizeof(grant_status_t))
 161
 162 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 163 {
 164         return &gnttab_list[(entry) / RPP][(entry) % RPP];
 165 }
 166 /* This can be used as an l-value */
 167 #define gnttab_entry(entry) (*__gnttab_entry(entry))
 168
 169 static int get_free_entries(unsigned count)
 170 {
 171         unsigned long flags;
 172         int ref, rc = 0;
 173         grant_ref_t head;
 174
 175         spin_lock_irqsave(&gnttab_list_lock, flags);
 176
 177         if ((gnttab_free_count < count) &&
 178             ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 179                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 180                 return rc;
 181         }
 182
 183         ref = head = gnttab_free_head;
 184         gnttab_free_count -= count;
 185         while (count-- > 1)
 186                 head = gnttab_entry(head);
 187         gnttab_free_head = gnttab_entry(head);
 188         gnttab_entry(head) = GNTTAB_LIST_END;
 189
 190         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 191
 192         return ref;
 193 }
 194
 195 static void do_free_callbacks(void)
 196 {
 197         struct gnttab_free_callback *callback, *next;
 198
 199         callback = gnttab_free_callback_list;
 200         gnttab_free_callback_list = NULL;
 201
 202         while (callback != NULL) {
 203                 next = callback->next;
 204                 if (gnttab_free_count >= callback->count) {
 205                         callback->next = NULL;
 206                         callback->fn(callback->arg);
 207                 } else {
 208                         callback->next = gnttab_free_callback_list;
 209                         gnttab_free_callback_list = callback;
 210                 }
 211                 callback = next;
 212         }
 213 }
 214
 215 static inline void check_free_callbacks(void)
 216 {
 217         if (unlikely(gnttab_free_callback_list))
 218                 do_free_callbacks();
 219 }
 220
 221 static void put_free_entry(grant_ref_t ref)
 222 {
 223         unsigned long flags;
 224         spin_lock_irqsave(&gnttab_list_lock, flags);
 225         gnttab_entry(ref) = gnttab_free_head;
 226         gnttab_free_head = ref;
 227         gnttab_free_count++;
 228         check_free_callbacks();
 229         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 230 }
 231
 232 /*
 233  * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 234  * Introducing a valid entry into the grant table:
 235  *  1. Write ent->domid.
 236  *  2. Write ent->frame:
 237  *      GTF_permit_access:   Frame to which access is permitted.
 238  *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 239  *                           frame, or zero if none.
 240  *  3. Write memory barrier (WMB).
 241  *  4. Write ent->flags, inc. valid type.
 242  */
 243 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 244                                    unsigned long frame, unsigned flags)
 245 {
 246         gnttab_shared.v1[ref].domid = domid;
 247         gnttab_shared.v1[ref].frame = frame;
 248         wmb();
 249         gnttab_shared.v1[ref].flags = flags;
 250 }
 251
 252 static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 253                                    unsigned long frame, unsigned flags)
 254 {
 255         gnttab_shared.v2[ref].hdr.domid = domid;
 256         gnttab_shared.v2[ref].full_page.frame = frame;
 257         wmb();
 258         gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 259 }
 260
 261 /*
 262  * Public grant-issuing interface functions
 263  */
 264 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 265                                      unsigned long frame, int readonly)
 266 {
 267         gnttab_interface->update_entry(ref, domid, frame,
 268                            GTF_permit_access | (readonly ? GTF_readonly : 0));
 269 }
 270 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 271
 272 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 273                                 int readonly)
 274 {
 275         int ref;
 276
 277         ref = get_free_entries(1);
 278         if (unlikely(ref < 0))
 279                 return -ENOSPC;
 280
 281         gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 282
 283         return ref;
 284 }
 285 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 286
 287 void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
 288                                     unsigned long frame, int flags,
 289                                     unsigned page_off,
 290                                     unsigned length)
 291 {
 292         gnttab_shared.v2[ref].sub_page.frame = frame;
 293         gnttab_shared.v2[ref].sub_page.page_off = page_off;
 294         gnttab_shared.v2[ref].sub_page.length = length;
 295         gnttab_shared.v2[ref].hdr.domid = domid;
 296         wmb();
 297         gnttab_shared.v2[ref].hdr.flags =
 298                                 GTF_permit_access | GTF_sub_page | flags;
 299 }
 300
 301 int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
 302                                             unsigned long frame, int flags,
 303                                             unsigned page_off,
 304                                             unsigned length)
 305 {
 306         if (flags & (GTF_accept_transfer | GTF_reading |
 307                      GTF_writing | GTF_transitive))
 308                 return -EPERM;
 309
 310         if (gnttab_interface->update_subpage_entry == NULL)
 311                 return -ENOSYS;
 312
 313         gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
 314                                                page_off, length);
 315
 316         return 0;
 317 }
 318 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
 319
 320 int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
 321                                         int flags, unsigned page_off,
 322                                         unsigned length)
 323 {
 324         int ref, rc;
 325
 326         ref = get_free_entries(1);
 327         if (unlikely(ref < 0))
 328                 return -ENOSPC;
 329
 330         rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
 331                                                      page_off, length);
 332         if (rc < 0) {
 333                 put_free_entry(ref);
 334                 return rc;
 335         }
 336
 337         return ref;
 338 }
 339 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
 340
 341 bool gnttab_subpage_grants_available(void)
 342 {
 343         return gnttab_interface->update_subpage_entry != NULL;
 344 }
 345 EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
 346
 347 void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
 348                                   int flags, domid_t trans_domid,
 349                                   grant_ref_t trans_gref)
 350 {
 351         gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
 352         gnttab_shared.v2[ref].transitive.gref = trans_gref;
 353         gnttab_shared.v2[ref].hdr.domid = domid;
 354         wmb();
 355         gnttab_shared.v2[ref].hdr.flags =
 356                                 GTF_permit_access | GTF_transitive | flags;
 357 }
 358
 359 int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
 360                                           int flags, domid_t trans_domid,
 361                                           grant_ref_t trans_gref)
 362 {
 363         if (flags & (GTF_accept_transfer | GTF_reading |
 364                      GTF_writing | GTF_sub_page))
 365                 return -EPERM;
 366
 367         if (gnttab_interface->update_trans_entry == NULL)
 368                 return -ENOSYS;
 369
 370         gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
 371                                              trans_gref);
 372
 373         return 0;
 374 }
 375 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
 376
 377 int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
 378                                       domid_t trans_domid,
 379                                       grant_ref_t trans_gref)
 380 {
 381         int ref, rc;
 382
 383         ref = get_free_entries(1);
 384         if (unlikely(ref < 0))
 385                 return -ENOSPC;
 386
 387         rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
 388                                                    trans_domid, trans_gref);
 389         if (rc < 0) {
 390                 put_free_entry(ref);
 391                 return rc;
 392         }
 393
 394         return ref;
 395 }
 396 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
 397
 398 bool gnttab_trans_grants_available(void)
 399 {
 400         return gnttab_interface->update_trans_entry != NULL;
 401 }
 402 EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
 403
 404 static int gnttab_query_foreign_access_v1(grant_ref_t ref)
 405 {
 406         return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
 407 }
 408
 409 static int gnttab_query_foreign_access_v2(grant_ref_t ref)
 410 {
 411         return grstatus[ref] & (GTF_reading|GTF_writing);
 412 }
 413
 414 int gnttab_query_foreign_access(grant_ref_t ref)
 415 {
 416         return gnttab_interface->query_foreign_access(ref);
 417 }
 418 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 419
 420 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
 421 {
 422         u16 flags, nflags;
 423         u16 *pflags;
 424
 425         pflags = &gnttab_shared.v1[ref].flags;
 426         nflags = *pflags;
 427         do {
 428                 flags = nflags;
 429                 if (flags & (GTF_reading|GTF_writing)) {
 430                         printk(KERN_ALERT "WARNING: g.e. still in use!\n");
 431                         return 0;
 432                 }
 433         } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
 434
 435         return 1;
 436 }
 437
 438 static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
 439 {
 440         gnttab_shared.v2[ref].hdr.flags = 0;
 441         mb();
 442         if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 443                 return 0;
 444         } else {
 445                 /* The read of grstatus needs to have acquire
 446                 semantics.  On x86, reads already have
 447                 that, and we just need to protect against
 448                 compiler reorderings.  On other
 449                 architectures we may need a full
 450                 barrier. */
 451 #ifdef CONFIG_X86
 452                 barrier();
 453 #else
 454                 mb();
 455 #endif
 456         }
 457
 458         return 1;
 459 }
 460
 461 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 462 {
 463         return gnttab_interface->end_foreign_access_ref(ref, readonly);
 464 }
 465 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 466
 467 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 468                                unsigned long page)
 469 {
 470         if (gnttab_end_foreign_access_ref(ref, readonly)) {
 471                 put_free_entry(ref);
 472                 if (page != 0)
 473                         free_page(page);
 474         } else {
 475                 /* XXX This needs to be fixed so that the ref and page are
 476                    placed on a list to be freed up later. */
 477                 printk(KERN_WARNING
 478                        "WARNING: leaking g.e. and page still in use!\n");
 479         }
 480 }
 481 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 482
 483 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 484 {
 485         int ref;
 486
 487         ref = get_free_entries(1);
 488         if (unlikely(ref < 0))
 489                 return -ENOSPC;
 490         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 491
 492         return ref;
 493 }
 494 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 495
 496 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 497                                        unsigned long pfn)
 498 {
 499         gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
 500 }
 501 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 502
 503 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
 504 {
 505         unsigned long frame;
 506         u16           flags;
 507         u16          *pflags;
 508
 509         pflags = &gnttab_shared.v1[ref].flags;
 510
 511         /*
 512          * If a transfer is not even yet started, try to reclaim the grant
 513          * reference and return failure (== 0).
 514          */
 515         while (!((flags = *pflags) & GTF_transfer_committed)) {
 516                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 517                         return 0;
 518                 cpu_relax();
 519         }
 520
 521         /* If a transfer is in progress then wait until it is completed. */
 522         while (!(flags & GTF_transfer_completed)) {
 523                 flags = *pflags;
 524                 cpu_relax();
 525         }
 526
 527         rmb();  /* Read the frame number /after/ reading completion status. */
 528         frame = gnttab_shared.v1[ref].frame;
 529         BUG_ON(frame == 0);
 530
 531         return frame;
 532 }
 533
 534 static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
 535 {
 536         unsigned long frame;
 537         u16           flags;
 538         u16          *pflags;
 539
 540         pflags = &gnttab_shared.v2[ref].hdr.flags;
 541
 542         /*
 543          * If a transfer is not even yet started, try to reclaim the grant
 544          * reference and return failure (== 0).
 545          */
 546         while (!((flags = *pflags) & GTF_transfer_committed)) {
 547                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 548                         return 0;
 549                 cpu_relax();
 550         }
 551
 552         /* If a transfer is in progress then wait until it is completed. */
 553         while (!(flags & GTF_transfer_completed)) {
 554                 flags = *pflags;
 555                 cpu_relax();
 556         }
 557
 558         rmb();  /* Read the frame number /after/ reading completion status. */
 559         frame = gnttab_shared.v2[ref].full_page.frame;
 560         BUG_ON(frame == 0);
 561
 562         return frame;
 563 }
 564
 565 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 566 {
 567         return gnttab_interface->end_foreign_transfer_ref(ref);
 568 }
 569 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 570
 571 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 572 {
 573         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 574         put_free_entry(ref);
 575         return frame;
 576 }
 577 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 578
 579 void gnttab_free_grant_reference(grant_ref_t ref)
 580 {
 581         put_free_entry(ref);
 582 }
 583 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 584
 585 void gnttab_free_grant_references(grant_ref_t head)
 586 {
 587         grant_ref_t ref;
 588         unsigned long flags;
 589         int count = 1;
 590         if (head == GNTTAB_LIST_END)
 591                 return;
 592         spin_lock_irqsave(&gnttab_list_lock, flags);
 593         ref = head;
 594         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 595                 ref = gnttab_entry(ref);
 596                 count++;
 597         }
 598         gnttab_entry(ref) = gnttab_free_head;
 599         gnttab_free_head = head;
 600         gnttab_free_count += count;
 601         check_free_callbacks();
 602         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 603 }
 604 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 605
 606 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 607 {
 608         int h = get_free_entries(count);
 609
 610         if (h < 0)
 611                 return -ENOSPC;
 612
 613         *head = h;
 614
 615         return 0;
 616 }
 617 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 618
 619 int gnttab_empty_grant_references(const grant_ref_t *private_head)
 620 {
 621         return (*private_head == GNTTAB_LIST_END);
 622 }
 623 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 624
 625 int gnttab_claim_grant_reference(grant_ref_t *private_head)
 626 {
 627         grant_ref_t g = *private_head;
 628         if (unlikely(g == GNTTAB_LIST_END))
 629                 return -ENOSPC;
 630         *private_head = gnttab_entry(g);
 631         return g;
 632 }
 633 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 634
 635 void gnttab_release_grant_reference(grant_ref_t *private_head,
 636                                     grant_ref_t release)
 637 {
 638         gnttab_entry(release) = *private_head;
 639         *private_head = release;
 640 }
 641 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 642
 643 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 644                                   void (*fn)(void *), void *arg, u16 count)
 645 {
 646         unsigned long flags;
 647         spin_lock_irqsave(&gnttab_list_lock, flags);
 648         if (callback->next)
 649                 goto out;
 650         callback->fn = fn;
 651         callback->arg = arg;
 652         callback->count = count;
 653         callback->next = gnttab_free_callback_list;
 654         gnttab_free_callback_list = callback;
 655         check_free_callbacks();
 656 out:
 657         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 658 }
 659 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 660
 661 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 662 {
 663         struct gnttab_free_callback **pcb;
 664         unsigned long flags;
 665
 666         spin_lock_irqsave(&gnttab_list_lock, flags);
 667         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 668                 if (*pcb == callback) {
 669                         *pcb = callback->next;
 670                         break;
 671                 }
 672         }
 673         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 674 }
 675 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 676
 677 static int grow_gnttab_list(unsigned int more_frames)
 678 {
 679         unsigned int new_nr_grant_frames, extra_entries, i;
 680         unsigned int nr_glist_frames, new_nr_glist_frames;
 681
 682         new_nr_grant_frames = nr_grant_frames + more_frames;
 683         extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
 684
 685         nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 686         new_nr_glist_frames =
 687                 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 688         for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 689                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 690                 if (!gnttab_list[i])
 691                         goto grow_nomem;
 692         }
 693
 694
 695         for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 696              i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
 697                 gnttab_entry(i) = i + 1;
 698
 699         gnttab_entry(i) = gnttab_free_head;
 700         gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 701         gnttab_free_count += extra_entries;
 702
 703         nr_grant_frames = new_nr_grant_frames;
 704
 705         check_free_callbacks();
 706
 707         return 0;
 708
 709 grow_nomem:
 710         for ( ; i >= nr_glist_frames; i--)
 711                 free_page((unsigned long) gnttab_list[i]);
 712         return -ENOMEM;
 713 }
 714
 715 static unsigned int __max_nr_grant_frames(void)
 716 {
 717         struct gnttab_query_size query;
 718         int rc;
 719
 720         query.dom = DOMID_SELF;
 721
 722         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 723         if ((rc < 0) || (query.status != GNTST_okay))
 724                 return 4; /* Legacy max supported number of frames */
 725
 726         return query.max_nr_frames;
 727 }
 728
 729 unsigned int gnttab_max_grant_frames(void)
 730 {
 731         unsigned int xen_max = __max_nr_grant_frames();
 732
 733         if (xen_max > boot_max_nr_grant_frames)
 734                 return boot_max_nr_grant_frames;
 735         return xen_max;
 736 }
 737 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 738
 739 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 740                     struct gnttab_map_grant_ref *kmap_ops,
 741                     struct page **pages, unsigned int count)
 742 {
 743         int i, ret;
 744         pte_t *pte;
 745         unsigned long mfn;
 746
 747         ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
 748         if (ret)
 749                 return ret;
 750
 751         if (xen_feature(XENFEAT_auto_translated_physmap))
 752                 return ret;
 753
 754         for (i = 0; i < count; i++) {
 755                 /* Do not add to override if the map failed. */
 756                 if (map_ops[i].status)
 757                         continue;
 758
 759                 if (map_ops[i].flags & GNTMAP_contains_pte) {
 760                         pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
 761                                 (map_ops[i].host_addr & ~PAGE_MASK));
 762                         mfn = pte_mfn(*pte);
 763                 } else {
 764                         mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
 765                 }
 766                 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
 767                                        &kmap_ops[i] : NULL);
 768                 if (ret)
 769                         return ret;
 770         }
 771
 772         return ret;
 773 }
 774 EXPORT_SYMBOL_GPL(gnttab_map_refs);
 775
 776 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 777                       struct page **pages, unsigned int count, bool clear_pte)
 778 {
 779         int i, ret;
 780
 781         ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
 782         if (ret)
 783                 return ret;
 784
 785         if (xen_feature(XENFEAT_auto_translated_physmap))
 786                 return ret;
 787
 788         for (i = 0; i < count; i++) {
 789                 ret = m2p_remove_override(pages[i], clear_pte);
 790                 if (ret)
 791                         return ret;
 792         }
 793
 794         return ret;
 795 }
 796 EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
 797
 798 static unsigned nr_status_frames(unsigned nr_grant_frames)
 799 {
 800         return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
 801 }
 802
 803 static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
 804 {
 805         int rc;
 806
 807         rc = arch_gnttab_map_shared(frames, nr_gframes,
 808                                     gnttab_max_grant_frames(),
 809                                     &gnttab_shared.addr);
 810         BUG_ON(rc);
 811
 812         return 0;
 813 }
 814
 815 static void gnttab_unmap_frames_v1(void)
 816 {
 817         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 818 }
 819
 820 static int gnttab_map_frames_v2(unsigned long *frames, unsigned int nr_gframes)
 821 {
 822         uint64_t *sframes;
 823         unsigned int nr_sframes;
 824         struct gnttab_get_status_frames getframes;
 825         int rc;
 826
 827         nr_sframes = nr_status_frames(nr_gframes);
 828
 829         /* No need for kzalloc as it is initialized in following hypercall
 830          * GNTTABOP_get_status_frames.
 831          */
 832         sframes = kmalloc(nr_sframes  * sizeof(uint64_t), GFP_ATOMIC);
 833         if (!sframes)
 834                 return -ENOMEM;
 835
 836         getframes.dom        = DOMID_SELF;
 837         getframes.nr_frames  = nr_sframes;
 838         set_xen_guest_handle(getframes.frame_list, sframes);
 839
 840         rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
 841                                        &getframes, 1);
 842         if (rc == -ENOSYS) {
 843                 kfree(sframes);
 844                 return -ENOSYS;
 845         }
 846
 847         BUG_ON(rc || getframes.status);
 848
 849         rc = arch_gnttab_map_status(sframes, nr_sframes,
 850                                     nr_status_frames(gnttab_max_grant_frames()),
 851                                     &grstatus);
 852         BUG_ON(rc);
 853         kfree(sframes);
 854
 855         rc = arch_gnttab_map_shared(frames, nr_gframes,
 856                                     gnttab_max_grant_frames(),
 857                                     &gnttab_shared.addr);
 858         BUG_ON(rc);
 859
 860         return 0;
 861 }
 862
 863 static void gnttab_unmap_frames_v2(void)
 864 {
 865         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 866         arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
 867 }
 868
 869 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 870 {
 871         struct gnttab_setup_table setup;
 872         unsigned long *frames;
 873         unsigned int nr_gframes = end_idx + 1;
 874         int rc;
 875
 876         if (xen_hvm_domain()) {
 877                 struct xen_add_to_physmap xatp;
 878                 unsigned int i = end_idx;
 879                 rc = 0;
 880                 /*
 881                  * Loop backwards, so that the first hypercall has the largest
 882                  * index, ensuring that the table will grow only once.
 883                  */
 884                 do {
 885                         xatp.domid = DOMID_SELF;
 886                         xatp.idx = i;
 887                         xatp.space = XENMAPSPACE_grant_table;
 888                         xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
 889                         rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 890                         if (rc != 0) {
 891                                 printk(KERN_WARNING
 892                                                 "grant table add_to_physmap failed, err=%d\n", rc);
 893                                 break;
 894                         }
 895                 } while (i-- > start_idx);
 896
 897                 return rc;
 898         }
 899
 900         /* No need for kzalloc as it is initialized in following hypercall
 901          * GNTTABOP_setup_table.
 902          */
 903         frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
 904         if (!frames)
 905                 return -ENOMEM;
 906
 907         setup.dom        = DOMID_SELF;
 908         setup.nr_frames  = nr_gframes;
 909         set_xen_guest_handle(setup.frame_list, frames);
 910
 911         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
 912         if (rc == -ENOSYS) {
 913                 kfree(frames);
 914                 return -ENOSYS;
 915         }
 916
 917         BUG_ON(rc || setup.status);
 918
 919         rc = gnttab_interface->map_frames(frames, nr_gframes);
 920
 921         kfree(frames);
 922
 923         return rc;
 924 }
 925
 926 static struct gnttab_ops gnttab_v1_ops = {
 927         .map_frames                     = gnttab_map_frames_v1,
 928         .unmap_frames                   = gnttab_unmap_frames_v1,
 929         .update_entry                   = gnttab_update_entry_v1,
 930         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v1,
 931         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v1,
 932         .query_foreign_access           = gnttab_query_foreign_access_v1,
 933 };
 934
 935 static struct gnttab_ops gnttab_v2_ops = {
 936         .map_frames                     = gnttab_map_frames_v2,
 937         .unmap_frames                   = gnttab_unmap_frames_v2,
 938         .update_entry                   = gnttab_update_entry_v2,
 939         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v2,
 940         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v2,
 941         .query_foreign_access           = gnttab_query_foreign_access_v2,
 942         .update_subpage_entry           = gnttab_update_subpage_entry_v2,
 943         .update_trans_entry             = gnttab_update_trans_entry_v2,
 944 };
 945
 946 static void gnttab_request_version(void)
 947 {
 948         int rc;
 949         struct gnttab_set_version gsv;
 950
 951         if (xen_hvm_domain())
 952                 gsv.version = 1;
 953         else
 954                 gsv.version = 2;
 955         rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
 956         if (rc == 0 && gsv.version == 2) {
 957                 grant_table_version = 2;
 958                 gnttab_interface = &gnttab_v2_ops;
 959         } else if (grant_table_version == 2) {
 960                 /*
 961                  * If we've already used version 2 features,
 962                  * but then suddenly discover that they're not
 963                  * available (e.g. migrating to an older
 964                  * version of Xen), almost unbounded badness
 965                  * can happen.
 966                  */
 967                 panic("we need grant tables version 2, but only version 1 is available");
 968         } else {
 969                 grant_table_version = 1;
 970                 gnttab_interface = &gnttab_v1_ops;
 971         }
 972         printk(KERN_INFO "Grant tables using version %d layout.\n",
 973                 grant_table_version);
 974 }
 975
 976 int gnttab_resume(void)
 977 {
 978         unsigned int max_nr_gframes;
 979
 980         gnttab_request_version();
 981         max_nr_gframes = gnttab_max_grant_frames();
 982         if (max_nr_gframes < nr_grant_frames)
 983                 return -ENOSYS;
 984
 985         if (xen_pv_domain())
 986                 return gnttab_map(0, nr_grant_frames - 1);
 987
 988         if (gnttab_shared.addr == NULL) {
 989                 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
 990                                                 PAGE_SIZE * max_nr_gframes);
 991                 if (gnttab_shared.addr == NULL) {
 992                         printk(KERN_WARNING
 993                                         "Failed to ioremap gnttab share frames!");
 994                         return -ENOMEM;
 995                 }
 996         }
 997
 998         gnttab_map(0, nr_grant_frames - 1);
 999
1000         return 0;
1001 }
1002
1003 int gnttab_suspend(void)
1004 {
1005         gnttab_interface->unmap_frames();
1006         return 0;
1007 }
1008
1009 static int gnttab_expand(unsigned int req_entries)
1010 {
1011         int rc;
1012         unsigned int cur, extra;
1013
1014         cur = nr_grant_frames;
1015         extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1016                  GREFS_PER_GRANT_FRAME);
1017         if (cur + extra > gnttab_max_grant_frames())
1018                 return -ENOSPC;
1019
1020         rc = gnttab_map(cur, cur + extra - 1);
1021         if (rc == 0)
1022                 rc = grow_gnttab_list(extra);
1023
1024         return rc;
1025 }
1026
1027 int gnttab_init(void)
1028 {
1029         int i;
1030         unsigned int max_nr_glist_frames, nr_glist_frames;
1031         unsigned int nr_init_grefs;
1032
1033         nr_grant_frames = 1;
1034         boot_max_nr_grant_frames = __max_nr_grant_frames();
1035
1036         /* Determine the maximum number of frames required for the
1037          * grant reference free list on the current hypervisor.
1038          */
1039         max_nr_glist_frames = (boot_max_nr_grant_frames *
1040                                GREFS_PER_GRANT_FRAME / RPP);
1041
1042         gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1043                               GFP_KERNEL);
1044         if (gnttab_list == NULL)
1045                 return -ENOMEM;
1046
1047         nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1048         for (i = 0; i < nr_glist_frames; i++) {
1049                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1050                 if (gnttab_list[i] == NULL)
1051                         goto ini_nomem;
1052         }
1053
1054         if (gnttab_resume() < 0)
1055                 return -ENODEV;
1056
1057         nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1058
1059         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1060                 gnttab_entry(i) = i + 1;
1061
1062         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1063         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1064         gnttab_free_head  = NR_RESERVED_ENTRIES;
1065
1066         printk("Grant table initialized\n");
1067         return 0;
1068
1069  ini_nomem:
1070         for (i--; i >= 0; i--)
1071                 free_page((unsigned long)gnttab_list[i]);
1072         kfree(gnttab_list);
1073         return -ENOMEM;
1074 }
1075 EXPORT_SYMBOL_GPL(gnttab_init);
1076
1077 static int __devinit __gnttab_init(void)
1078 {
1079         /* Delay grant-table initialization in the PV on HVM case */
1080         if (xen_hvm_domain())
1081                 return 0;
1082
1083         if (!xen_pv_domain())
1084                 return -ENODEV;
1085
1086         return gnttab_init();
1087 }
1088
1089 core_initcall(__gnttab_init);