1 /******************************************************************************
4 * Granting foreign access to our memory reservation.
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 #include <linux/export.h>
35 #include <linux/slab.h>
36 #include <linux/sched.h>
38 #include <linux/seqlock.h>
39 #include <xen/interface/xen.h>
40 #include <xen/gnttab.h>
41 #include <asm/pgtable.h>
42 #include <asm/uaccess.h>
43 #include <asm/cmpxchg.h>
45 #include <xen/interface/memory.h>
46 #include <asm/gnttab_dma.h>
48 #ifdef HAVE_XEN_PLATFORM_COMPAT_H
49 #include <xen/platform-compat.h>
52 /* External tools reserve first few grant table entries. */
53 #define NR_RESERVED_ENTRIES 8
54 #define GNTTAB_LIST_END 0xffffffff
55 #define ENTRIES_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
57 static grant_ref_t **gnttab_list;
58 static unsigned int nr_grant_frames;
59 static unsigned int boot_max_nr_grant_frames;
60 static int gnttab_free_count;
61 static grant_ref_t gnttab_free_head;
62 static DEFINE_SPINLOCK(gnttab_list_lock);
64 static struct grant_entry *shared;
66 static struct gnttab_free_callback *gnttab_free_callback_list;
68 static int gnttab_expand(unsigned int req_entries);
70 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
71 #define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
73 #define nr_freelist_frames(grant_frames) \
74 (((grant_frames) * ENTRIES_PER_GRANT_FRAME + RPP - 1) / RPP)
76 static int get_free_entries(int count)
82 spin_lock_irqsave(&gnttab_list_lock, flags);
84 if ((gnttab_free_count < count) &&
85 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
86 spin_unlock_irqrestore(&gnttab_list_lock, flags);
90 ref = head = gnttab_free_head;
91 gnttab_free_count -= count;
93 head = gnttab_entry(head);
94 gnttab_free_head = gnttab_entry(head);
95 gnttab_entry(head) = GNTTAB_LIST_END;
97 spin_unlock_irqrestore(&gnttab_list_lock, flags);
102 #define get_free_entry() get_free_entries(1)
104 static void do_free_callbacks(void)
106 struct gnttab_free_callback *callback, *next;
108 callback = gnttab_free_callback_list;
109 gnttab_free_callback_list = NULL;
111 while (callback != NULL) {
112 next = callback->next;
113 if (gnttab_free_count >= callback->count) {
114 callback->next = NULL;
115 callback->queued = 0;
116 callback->fn(callback->arg);
118 callback->next = gnttab_free_callback_list;
119 gnttab_free_callback_list = callback;
125 static inline void check_free_callbacks(void)
127 if (unlikely(gnttab_free_callback_list))
131 static void put_free_entry(grant_ref_t ref)
134 spin_lock_irqsave(&gnttab_list_lock, flags);
135 gnttab_entry(ref) = gnttab_free_head;
136 gnttab_free_head = ref;
138 check_free_callbacks();
139 spin_unlock_irqrestore(&gnttab_list_lock, flags);
143 * Public grant-issuing interface functions
146 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
151 if (unlikely((ref = get_free_entry()) < 0))
154 shared[ref].frame = frame;
155 shared[ref].domid = domid;
157 BUG_ON(flags & (GTF_accept_transfer | GTF_reading | GTF_writing));
158 shared[ref].flags = GTF_permit_access | flags;
162 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
164 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
165 unsigned long frame, int flags)
167 shared[ref].frame = frame;
168 shared[ref].domid = domid;
170 BUG_ON(flags & (GTF_accept_transfer | GTF_reading | GTF_writing));
171 shared[ref].flags = GTF_permit_access | flags;
173 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
176 int gnttab_query_foreign_access(grant_ref_t ref)
180 nflags = shared[ref].flags;
182 return (nflags & (GTF_reading|GTF_writing));
184 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
186 int gnttab_end_foreign_access_ref(grant_ref_t ref)
190 nflags = shared[ref].flags;
192 if ((flags = nflags) & (GTF_reading|GTF_writing)) {
193 printk(KERN_DEBUG "WARNING: g.e. still in use!\n");
196 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) !=
201 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
203 void gnttab_end_foreign_access(grant_ref_t ref, unsigned long page)
205 if (gnttab_end_foreign_access_ref(ref)) {
210 /* XXX This needs to be fixed so that the ref and page are
211 placed on a list to be freed up later. */
213 "WARNING: leaking g.e. and page still in use!\n");
216 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
218 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
222 if (unlikely((ref = get_free_entry()) < 0))
224 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
228 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
230 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
233 shared[ref].frame = pfn;
234 shared[ref].domid = domid;
236 shared[ref].flags = GTF_accept_transfer;
238 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
240 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
246 * If a transfer is not even yet started, try to reclaim the grant
247 * reference and return failure (== 0).
249 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
250 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags)
255 /* If a transfer is in progress then wait until it is completed. */
256 while (!(flags & GTF_transfer_completed)) {
257 flags = shared[ref].flags;
261 /* Read the frame number /after/ reading completion status. */
263 frame = shared[ref].frame;
268 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
270 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
272 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
276 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
278 void gnttab_free_grant_reference(grant_ref_t ref)
282 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
284 void gnttab_free_grant_references(grant_ref_t head)
289 if (head == GNTTAB_LIST_END)
291 spin_lock_irqsave(&gnttab_list_lock, flags);
293 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
294 ref = gnttab_entry(ref);
297 gnttab_entry(ref) = gnttab_free_head;
298 gnttab_free_head = head;
299 gnttab_free_count += count;
300 check_free_callbacks();
301 spin_unlock_irqrestore(&gnttab_list_lock, flags);
303 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
305 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
307 int h = get_free_entries(count);
316 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
318 int gnttab_empty_grant_references(const grant_ref_t *private_head)
320 return (*private_head == GNTTAB_LIST_END);
322 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
324 int gnttab_claim_grant_reference(grant_ref_t *private_head)
326 grant_ref_t g = *private_head;
327 if (unlikely(g == GNTTAB_LIST_END))
329 *private_head = gnttab_entry(g);
332 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
334 void gnttab_release_grant_reference(grant_ref_t *private_head,
337 gnttab_entry(release) = *private_head;
338 *private_head = release;
340 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
342 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
343 void (*fn)(void *), void *arg, u16 count)
346 spin_lock_irqsave(&gnttab_list_lock, flags);
347 if (callback->queued)
351 callback->count = count;
352 callback->queued = 1;
353 callback->next = gnttab_free_callback_list;
354 gnttab_free_callback_list = callback;
355 check_free_callbacks();
357 spin_unlock_irqrestore(&gnttab_list_lock, flags);
359 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
361 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
363 struct gnttab_free_callback **pcb;
366 spin_lock_irqsave(&gnttab_list_lock, flags);
367 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
368 if (*pcb == callback) {
369 *pcb = callback->next;
370 callback->queued = 0;
374 spin_unlock_irqrestore(&gnttab_list_lock, flags);
376 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
378 static int grow_gnttab_list(unsigned int more_frames)
380 unsigned int new_nr_grant_frames, extra_entries, i;
381 unsigned int nr_glist_frames, new_nr_glist_frames;
383 new_nr_grant_frames = nr_grant_frames + more_frames;
384 extra_entries = more_frames * ENTRIES_PER_GRANT_FRAME;
386 nr_glist_frames = nr_freelist_frames(nr_grant_frames);
387 new_nr_glist_frames = nr_freelist_frames(new_nr_grant_frames);
388 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
389 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
394 for (i = ENTRIES_PER_GRANT_FRAME * nr_grant_frames;
395 i < ENTRIES_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
396 gnttab_entry(i) = i + 1;
398 gnttab_entry(i) = gnttab_free_head;
399 gnttab_free_head = ENTRIES_PER_GRANT_FRAME * nr_grant_frames;
400 gnttab_free_count += extra_entries;
402 nr_grant_frames = new_nr_grant_frames;
404 check_free_callbacks();
409 for ( ; i >= nr_glist_frames; i--)
410 free_page((unsigned long) gnttab_list[i]);
414 static unsigned int __max_nr_grant_frames(void)
416 struct gnttab_query_size query;
419 query.dom = DOMID_SELF;
421 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
422 if ((rc < 0) || (query.status != GNTST_okay))
423 return 4; /* Legacy max supported number of frames */
425 return query.max_nr_frames;
428 static inline unsigned int max_nr_grant_frames(void)
430 unsigned int xen_max = __max_nr_grant_frames();
432 if (xen_max > boot_max_nr_grant_frames)
433 return boot_max_nr_grant_frames;
440 static int map_pte_fn(pte_t *pte, struct page *pmd_page,
441 unsigned long addr, void *data)
443 unsigned long **frames = (unsigned long **)data;
445 set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL));
450 #ifdef CONFIG_PM_SLEEP
451 static int unmap_pte_fn(pte_t *pte, struct page *pmd_page,
452 unsigned long addr, void *data)
455 set_pte_at(&init_mm, addr, pte, __pte(0));
460 void *arch_gnttab_alloc_shared(unsigned long *frames)
462 struct vm_struct *area;
463 area = alloc_vm_area(PAGE_SIZE * max_nr_grant_frames(), NULL);
464 BUG_ON(area == NULL);
467 #endif /* CONFIG_X86 */
469 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
471 struct gnttab_setup_table setup;
472 unsigned long *frames;
473 unsigned int nr_gframes = end_idx + 1;
476 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
480 setup.dom = DOMID_SELF;
481 setup.nr_frames = nr_gframes;
482 set_xen_guest_handle(setup.frame_list, frames);
484 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
490 BUG_ON(rc || setup.status != GNTST_okay);
493 shared = arch_gnttab_alloc_shared(frames);
496 rc = apply_to_page_range(&init_mm, (unsigned long)shared,
497 PAGE_SIZE * nr_gframes,
498 map_pte_fn, &frames);
500 frames -= nr_gframes; /* adjust after map_pte_fn() */
501 #endif /* CONFIG_X86 */
508 #if defined(CONFIG_XEN_BACKEND) || defined(CONFIG_XEN_BACKEND_MODULE)
510 static DEFINE_SEQLOCK(gnttab_dma_lock);
512 static void gnttab_page_free(struct page *page, unsigned int order)
515 ClearPageForeign(page);
516 gnttab_reset_grant_page(page);
517 ClearPageReserved(page);
522 * Must not be called with IRQs off. This should only be used on the
525 * Copy a foreign granted page to local memory.
527 int gnttab_copy_grant_page(grant_ref_t ref, struct page **pagep)
529 struct gnttab_unmap_and_replace unmap;
532 struct page *new_page;
541 if (!get_page_unless_zero(page))
545 new_page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
549 new_addr = page_address(new_page);
550 addr = page_address(page);
551 copy_page(new_addr, addr);
553 pfn = page_to_pfn(page);
554 mfn = pfn_to_mfn(pfn);
555 new_mfn = virt_to_mfn(new_addr);
557 write_seqlock_bh(&gnttab_dma_lock);
559 /* Make seq visible before checking page_mapped. */
562 /* Has the page been DMA-mapped? */
563 if (unlikely(page_mapped(page))) {
564 write_sequnlock_bh(&gnttab_dma_lock);
570 if (!xen_feature(XENFEAT_auto_translated_physmap))
571 set_phys_to_machine(pfn, new_mfn);
573 gnttab_set_replace_op(&unmap, (unsigned long)addr,
574 (unsigned long)new_addr, ref);
576 err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_and_replace,
579 BUG_ON(unmap.status != GNTST_okay);
581 write_sequnlock_bh(&gnttab_dma_lock);
583 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
584 set_phys_to_machine(page_to_pfn(new_page), INVALID_P2M_ENTRY);
586 mmu.ptr = (new_mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
588 err = HYPERVISOR_mmu_update(&mmu, 1, NULL, DOMID_SELF);
592 new_page->mapping = page->mapping;
593 new_page->index = page->index;
594 set_bit(PG_foreign, &new_page->flags);
595 if (PageReserved(page))
596 SetPageReserved(new_page);
599 SetPageForeign(page, gnttab_page_free);
600 page->mapping = NULL;
606 EXPORT_SYMBOL_GPL(gnttab_copy_grant_page);
608 void gnttab_reset_grant_page(struct page *page)
610 init_page_count(page);
611 reset_page_mapcount(page);
613 EXPORT_SYMBOL_GPL(gnttab_reset_grant_page);
616 * Keep track of foreign pages marked as PageForeign so that we don't
617 * return them to the remote domain prematurely.
619 * PageForeign pages are pinned down by increasing their mapcount.
621 * All other pages are simply returned as is.
623 void __gnttab_dma_map_page(struct page *page)
627 if (!is_running_on_xen() || !PageForeign(page))
631 seq = read_seqbegin(&gnttab_dma_lock);
633 if (gnttab_dma_local_pfn(page))
636 atomic_set(&page->_mapcount, 0);
638 /* Make _mapcount visible before read_seqretry. */
640 } while (unlikely(read_seqretry(&gnttab_dma_lock, seq)));
643 #endif /* CONFIG_XEN_BACKEND */
645 #ifdef __HAVE_ARCH_PTE_SPECIAL
647 static unsigned int GNTMAP_pte_special;
649 bool gnttab_pre_map_adjust(unsigned int cmd, struct gnttab_map_grant_ref *map,
654 if (unlikely(cmd != GNTTABOP_map_grant_ref))
657 for (i = 0; i < count; ++i, ++map) {
658 if (!(map->flags & GNTMAP_host_map)
659 || !(map->flags & GNTMAP_application_map))
661 if (GNTMAP_pte_special)
662 map->flags |= GNTMAP_pte_special;
664 BUG_ON(xen_feature(XENFEAT_auto_translated_physmap));
671 EXPORT_SYMBOL(gnttab_pre_map_adjust);
673 #if CONFIG_XEN_COMPAT < 0x030400
674 int gnttab_post_map_adjust(const struct gnttab_map_grant_ref *map, unsigned int count)
679 for (i = 0; i < count && rc == 0; ++i, ++map) {
682 if (!(map->flags & GNTMAP_host_map)
683 || !(map->flags & GNTMAP_application_map))
687 pte = __pte_ma((map->dev_bus_addr | _PAGE_PRESENT | _PAGE_USER
688 | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_NX
690 & __supported_pte_mask);
692 #error Architecture not yet supported.
694 if (!(map->flags & GNTMAP_readonly))
695 pte = pte_mkwrite(pte);
697 if (map->flags & GNTMAP_contains_pte) {
700 u.ptr = map->host_addr;
701 u.val = __pte_val(pte);
702 rc = HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF);
704 rc = HYPERVISOR_update_va_mapping(map->host_addr, pte, 0);
709 EXPORT_SYMBOL(gnttab_post_map_adjust);
712 #endif /* __HAVE_ARCH_PTE_SPECIAL */
714 int gnttab_resume(void)
716 if (max_nr_grant_frames() < nr_grant_frames)
718 return gnttab_map(0, nr_grant_frames - 1);
721 #ifdef CONFIG_PM_SLEEP
722 #include <linux/syscore_ops.h>
725 static int gnttab_suspend(void)
727 apply_to_page_range(&init_mm, (unsigned long)shared,
728 PAGE_SIZE * nr_grant_frames,
733 #define gnttab_suspend NULL
736 static void _gnttab_resume(void)
742 static struct syscore_ops gnttab_syscore_ops = {
743 .resume = _gnttab_resume,
744 .suspend = gnttab_suspend,
748 #else /* !CONFIG_XEN */
750 #include <platform-pci.h>
752 static unsigned long resume_frames;
754 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
756 struct xen_add_to_physmap xatp;
757 unsigned int i = end_idx;
759 /* Loop backwards, so that the first hypercall has the largest index,
760 * ensuring that the table will grow only once.
763 xatp.domid = DOMID_SELF;
765 xatp.space = XENMAPSPACE_grant_table;
766 xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
767 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
769 } while (i-- > start_idx);
774 int gnttab_resume(void)
776 unsigned int max_nr_gframes, nr_gframes;
778 nr_gframes = nr_grant_frames;
779 max_nr_gframes = max_nr_grant_frames();
780 if (max_nr_gframes < nr_gframes)
783 if (!resume_frames) {
784 resume_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
785 shared = ioremap(resume_frames, PAGE_SIZE * max_nr_gframes);
786 if (shared == NULL) {
787 pr_warning("error to ioremap gnttab share frames\n");
792 gnttab_map(0, nr_gframes - 1);
797 #endif /* !CONFIG_XEN */
799 static int gnttab_expand(unsigned int req_entries)
802 unsigned int cur, extra;
804 cur = nr_grant_frames;
805 extra = ((req_entries + (ENTRIES_PER_GRANT_FRAME-1)) /
806 ENTRIES_PER_GRANT_FRAME);
807 if (cur + extra > max_nr_grant_frames())
810 if ((rc = gnttab_map(cur, cur + extra - 1)) == 0)
811 rc = grow_gnttab_list(extra);
824 unsigned int max_nr_glist_frames, nr_glist_frames;
825 unsigned int nr_init_grefs;
827 if (!is_running_on_xen())
831 boot_max_nr_grant_frames = __max_nr_grant_frames();
833 /* Determine the maximum number of frames required for the
834 * grant reference free list on the current hypervisor.
836 max_nr_glist_frames = nr_freelist_frames(boot_max_nr_grant_frames);
838 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
840 if (gnttab_list == NULL)
843 nr_glist_frames = nr_freelist_frames(nr_grant_frames);
844 for (i = 0; i < nr_glist_frames; i++) {
845 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
846 if (gnttab_list[i] == NULL)
850 if (gnttab_resume() < 0)
853 nr_init_grefs = nr_grant_frames * ENTRIES_PER_GRANT_FRAME;
855 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
856 gnttab_entry(i) = i + 1;
858 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
859 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
860 gnttab_free_head = NR_RESERVED_ENTRIES;
862 #if defined(CONFIG_XEN) && defined(__HAVE_ARCH_PTE_SPECIAL)
863 if (!xen_feature(XENFEAT_auto_translated_physmap)
864 && xen_feature(XENFEAT_gnttab_map_avail_bits)) {
866 GNTMAP_pte_special = (__pte_val(pte_mkspecial(__pte_ma(0)))
867 >> _PAGE_BIT_UNUSED1) << _GNTMAP_guest_avail0;
869 #error Architecture not yet supported.
874 #if defined(CONFIG_XEN) && defined(CONFIG_PM_SLEEP)
875 if (!is_initial_xendomain())
876 register_syscore_ops(&gnttab_syscore_ops);
882 for (i--; i >= 0; i--)
883 free_page((unsigned long)gnttab_list[i]);
889 core_initcall(gnttab_init);