drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/export.h>
  28 #include <linux/slab.h>
  29 #include <linux/irq.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/pci.h>
  33 #include <linux/dmar.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/mempool.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <linux/memblock.h>
  45 #include <asm/cacheflush.h>
  46 #include <asm/iommu.h>
  47
  48 #define ROOT_SIZE               VTD_PAGE_SIZE
  49 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  50
  51 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  52 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  53 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  54
  55 #define IOAPIC_RANGE_START      (0xfee00000)
  56 #define IOAPIC_RANGE_END        (0xfeefffff)
  57 #define IOVA_START_ADDR         (0x1000)
  58
  59 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  60
  61 #define MAX_AGAW_WIDTH 64
  62
  63 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  64 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  65
  66 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  67    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  68 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  69                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  70 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  71
  72 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  73 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  74 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  75
  76 /* page table handling */
  77 #define LEVEL_STRIDE            (9)
  78 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  79
  80 /*
  81  * This bitmap is used to advertise the page sizes our hardware support
  82  * to the IOMMU core, which will then use this information to split
  83  * physically contiguous memory regions it is mapping into page sizes
  84  * that we support.
  85  *
  86  * Traditionally the IOMMU core just handed us the mappings directly,
  87  * after making sure the size is an order of a 4KiB page and that the
  88  * mapping has natural alignment.
  89  *
  90  * To retain this behavior, we currently advertise that we support
  91  * all page sizes that are an order of 4KiB.
  92  *
  93  * If at some point we'd like to utilize the IOMMU core's new behavior,
  94  * we could change this to advertise the real page sizes we support.
  95  */
  96 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
  97
  98 static inline int agaw_to_level(int agaw)
  99 {
 100         return agaw + 2;
 101 }
 102
 103 static inline int agaw_to_width(int agaw)
 104 {
 105         return 30 + agaw * LEVEL_STRIDE;
 106 }
 107
 108 static inline int width_to_agaw(int width)
 109 {
 110         return (width - 30) / LEVEL_STRIDE;
 111 }
 112
 113 static inline unsigned int level_to_offset_bits(int level)
 114 {
 115         return (level - 1) * LEVEL_STRIDE;
 116 }
 117
 118 static inline int pfn_level_offset(unsigned long pfn, int level)
 119 {
 120         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 121 }
 122
 123 static inline unsigned long level_mask(int level)
 124 {
 125         return -1UL << level_to_offset_bits(level);
 126 }
 127
 128 static inline unsigned long level_size(int level)
 129 {
 130         return 1UL << level_to_offset_bits(level);
 131 }
 132
 133 static inline unsigned long align_to_level(unsigned long pfn, int level)
 134 {
 135         return (pfn + level_size(level) - 1) & level_mask(level);
 136 }
 137
 138 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 139 {
 140         return  1 << ((lvl - 1) * LEVEL_STRIDE);
 141 }
 142
 143 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 144    are never going to work. */
 145 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 146 {
 147         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 148 }
 149
 150 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 151 {
 152         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 153 }
 154 static inline unsigned long page_to_dma_pfn(struct page *pg)
 155 {
 156         return mm_to_dma_pfn(page_to_pfn(pg));
 157 }
 158 static inline unsigned long virt_to_dma_pfn(void *p)
 159 {
 160         return page_to_dma_pfn(virt_to_page(p));
 161 }
 162
 163 /* global iommu list, set NULL for ignored DMAR units */
 164 static struct intel_iommu **g_iommus;
 165
 166 static void __init check_tylersburg_isoch(void);
 167 static int rwbf_quirk;
 168
 169 /*
 170  * set to 1 to panic kernel if can't successfully enable VT-d
 171  * (used when kernel is launched w/ TXT)
 172  */
 173 static int force_on = 0;
 174
 175 /*
 176  * 0: Present
 177  * 1-11: Reserved
 178  * 12-63: Context Ptr (12 - (haw-1))
 179  * 64-127: Reserved
 180  */
 181 struct root_entry {
 182         u64     val;
 183         u64     rsvd1;
 184 };
 185 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 186 static inline bool root_present(struct root_entry *root)
 187 {
 188         return (root->val & 1);
 189 }
 190 static inline void set_root_present(struct root_entry *root)
 191 {
 192         root->val |= 1;
 193 }
 194 static inline void set_root_value(struct root_entry *root, unsigned long value)
 195 {
 196         root->val |= value & VTD_PAGE_MASK;
 197 }
 198
 199 static inline struct context_entry *
 200 get_context_addr_from_root(struct root_entry *root)
 201 {
 202         return (struct context_entry *)
 203                 (root_present(root)?phys_to_virt(
 204                 root->val & VTD_PAGE_MASK) :
 205                 NULL);
 206 }
 207
 208 /*
 209  * low 64 bits:
 210  * 0: present
 211  * 1: fault processing disable
 212  * 2-3: translation type
 213  * 12-63: address space root
 214  * high 64 bits:
 215  * 0-2: address width
 216  * 3-6: aval
 217  * 8-23: domain id
 218  */
 219 struct context_entry {
 220         u64 lo;
 221         u64 hi;
 222 };
 223
 224 static inline bool context_present(struct context_entry *context)
 225 {
 226         return (context->lo & 1);
 227 }
 228 static inline void context_set_present(struct context_entry *context)
 229 {
 230         context->lo |= 1;
 231 }
 232
 233 static inline void context_set_fault_enable(struct context_entry *context)
 234 {
 235         context->lo &= (((u64)-1) << 2) | 1;
 236 }
 237
 238 static inline void context_set_translation_type(struct context_entry *context,
 239                                                 unsigned long value)
 240 {
 241         context->lo &= (((u64)-1) << 4) | 3;
 242         context->lo |= (value & 3) << 2;
 243 }
 244
 245 static inline void context_set_address_root(struct context_entry *context,
 246                                             unsigned long value)
 247 {
 248         context->lo |= value & VTD_PAGE_MASK;
 249 }
 250
 251 static inline void context_set_address_width(struct context_entry *context,
 252                                              unsigned long value)
 253 {
 254         context->hi |= value & 7;
 255 }
 256
 257 static inline void context_set_domain_id(struct context_entry *context,
 258                                          unsigned long value)
 259 {
 260         context->hi |= (value & ((1 << 16) - 1)) << 8;
 261 }
 262
 263 static inline void context_clear_entry(struct context_entry *context)
 264 {
 265         context->lo = 0;
 266         context->hi = 0;
 267 }
 268
 269 /*
 270  * 0: readable
 271  * 1: writable
 272  * 2-6: reserved
 273  * 7: super page
 274  * 8-10: available
 275  * 11: snoop behavior
 276  * 12-63: Host physcial address
 277  */
 278 struct dma_pte {
 279         u64 val;
 280 };
 281
 282 static inline void dma_clear_pte(struct dma_pte *pte)
 283 {
 284         pte->val = 0;
 285 }
 286
 287 static inline void dma_set_pte_readable(struct dma_pte *pte)
 288 {
 289         pte->val |= DMA_PTE_READ;
 290 }
 291
 292 static inline void dma_set_pte_writable(struct dma_pte *pte)
 293 {
 294         pte->val |= DMA_PTE_WRITE;
 295 }
 296
 297 static inline void dma_set_pte_snp(struct dma_pte *pte)
 298 {
 299         pte->val |= DMA_PTE_SNP;
 300 }
 301
 302 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 303 {
 304         pte->val = (pte->val & ~3) | (prot & 3);
 305 }
 306
 307 static inline u64 dma_pte_addr(struct dma_pte *pte)
 308 {
 309 #ifdef CONFIG_64BIT
 310         return pte->val & VTD_PAGE_MASK;
 311 #else
 312         /* Must have a full atomic 64-bit read */
 313         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 314 #endif
 315 }
 316
 317 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 318 {
 319         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 320 }
 321
 322 static inline bool dma_pte_present(struct dma_pte *pte)
 323 {
 324         return (pte->val & 3) != 0;
 325 }
 326
 327 static inline bool dma_pte_superpage(struct dma_pte *pte)
 328 {
 329         return (pte->val & (1 << 7));
 330 }
 331
 332 static inline int first_pte_in_page(struct dma_pte *pte)
 333 {
 334         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 335 }
 336
 337 /*
 338  * This domain is a statically identity mapping domain.
 339  *      1. This domain creats a static 1:1 mapping to all usable memory.
 340  *      2. It maps to each iommu if successful.
 341  *      3. Each iommu mapps to this domain if successful.
 342  */
 343 static struct dmar_domain *si_domain;
 344 static int hw_pass_through = 1;
 345
 346 /* devices under the same p2p bridge are owned in one domain */
 347 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 348
 349 /* domain represents a virtual machine, more than one devices
 350  * across iommus may be owned in one domain, e.g. kvm guest.
 351  */
 352 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 353
 354 /* si_domain contains mulitple devices */
 355 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 356
 357 /* define the limit of IOMMUs supported in each domain */
 358 #ifdef  CONFIG_X86
 359 # define        IOMMU_UNITS_SUPPORTED   MAX_IO_APICS
 360 #else
 361 # define        IOMMU_UNITS_SUPPORTED   64
 362 #endif
 363
 364 struct dmar_domain {
 365         int     id;                     /* domain id */
 366         int     nid;                    /* node id */
 367         DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
 368                                         /* bitmap of iommus this domain uses*/
 369
 370         struct list_head devices;       /* all devices' list */
 371         struct iova_domain iovad;       /* iova's that belong to this domain */
 372
 373         struct dma_pte  *pgd;           /* virtual address */
 374         int             gaw;            /* max guest address width */
 375
 376         /* adjusted guest address width, 0 is level 2 30-bit */
 377         int             agaw;
 378
 379         int             flags;          /* flags to find out type of domain */
 380
 381         int             iommu_coherency;/* indicate coherency of iommu access */
 382         int             iommu_snooping; /* indicate snooping control feature*/
 383         int             iommu_count;    /* reference count of iommu */
 384         int             iommu_superpage;/* Level of superpages supported:
 385                                            0 == 4KiB (no superpages), 1 == 2MiB,
 386                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 387         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 388         u64             max_addr;       /* maximum mapped address */
 389 };
 390
 391 /* PCI domain-device relationship */
 392 struct device_domain_info {
 393         struct list_head link;  /* link to domain siblings */
 394         struct list_head global; /* link to global list */
 395         int segment;            /* PCI domain */
 396         u8 bus;                 /* PCI bus number */
 397         u8 devfn;               /* PCI devfn number */
 398         struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
 399         struct intel_iommu *iommu; /* IOMMU used by this device */
 400         struct dmar_domain *domain; /* pointer to domain */
 401 };
 402
 403 static void flush_unmaps_timeout(unsigned long data);
 404
 405 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 406
 407 #define HIGH_WATER_MARK 250
 408 struct deferred_flush_tables {
 409         int next;
 410         struct iova *iova[HIGH_WATER_MARK];
 411         struct dmar_domain *domain[HIGH_WATER_MARK];
 412 };
 413
 414 static struct deferred_flush_tables *deferred_flush;
 415
 416 /* bitmap for indexing intel_iommus */
 417 static int g_num_of_iommus;
 418
 419 static DEFINE_SPINLOCK(async_umap_flush_lock);
 420 static LIST_HEAD(unmaps_to_do);
 421
 422 static int timer_on;
 423 static long list_size;
 424
 425 static void domain_remove_dev_info(struct dmar_domain *domain);
 426
 427 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 428 int dmar_disabled = 0;
 429 #else
 430 int dmar_disabled = 1;
 431 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 432
 433 int intel_iommu_enabled = 0;
 434 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 435
 436 static int dmar_map_gfx = 1;
 437 static int dmar_forcedac;
 438 static int intel_iommu_strict;
 439 static int intel_iommu_superpage = 1;
 440
 441 int intel_iommu_gfx_mapped;
 442 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 443
 444 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 445 static DEFINE_SPINLOCK(device_domain_lock);
 446 static LIST_HEAD(device_domain_list);
 447
 448 static struct iommu_ops intel_iommu_ops;
 449
 450 static int __init intel_iommu_setup(char *str)
 451 {
 452         if (!str)
 453                 return -EINVAL;
 454         while (*str) {
 455                 if (!strncmp(str, "on", 2)) {
 456                         dmar_disabled = 0;
 457                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 458                 } else if (!strncmp(str, "off", 3)) {
 459                         dmar_disabled = 1;
 460                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 461                 } else if (!strncmp(str, "igfx_off", 8)) {
 462                         dmar_map_gfx = 0;
 463                         printk(KERN_INFO
 464                                 "Intel-IOMMU: disable GFX device mapping\n");
 465                 } else if (!strncmp(str, "forcedac", 8)) {
 466                         printk(KERN_INFO
 467                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 468                         dmar_forcedac = 1;
 469                 } else if (!strncmp(str, "strict", 6)) {
 470                         printk(KERN_INFO
 471                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 472                         intel_iommu_strict = 1;
 473                 } else if (!strncmp(str, "sp_off", 6)) {
 474                         printk(KERN_INFO
 475                                 "Intel-IOMMU: disable supported super page\n");
 476                         intel_iommu_superpage = 0;
 477                 }
 478
 479                 str += strcspn(str, ",");
 480                 while (*str == ',')
 481                         str++;
 482         }
 483         return 0;
 484 }
 485 __setup("intel_iommu=", intel_iommu_setup);
 486
 487 static struct kmem_cache *iommu_domain_cache;
 488 static struct kmem_cache *iommu_devinfo_cache;
 489 static struct kmem_cache *iommu_iova_cache;
 490
 491 static inline void *alloc_pgtable_page(int node)
 492 {
 493         struct page *page;
 494         void *vaddr = NULL;
 495
 496         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 497         if (page)
 498                 vaddr = page_address(page);
 499         return vaddr;
 500 }
 501
 502 static inline void free_pgtable_page(void *vaddr)
 503 {
 504         free_page((unsigned long)vaddr);
 505 }
 506
 507 static inline void *alloc_domain_mem(void)
 508 {
 509         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 510 }
 511
 512 static void free_domain_mem(void *vaddr)
 513 {
 514         kmem_cache_free(iommu_domain_cache, vaddr);
 515 }
 516
 517 static inline void * alloc_devinfo_mem(void)
 518 {
 519         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 520 }
 521
 522 static inline void free_devinfo_mem(void *vaddr)
 523 {
 524         kmem_cache_free(iommu_devinfo_cache, vaddr);
 525 }
 526
 527 struct iova *alloc_iova_mem(void)
 528 {
 529         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 530 }
 531
 532 void free_iova_mem(struct iova *iova)
 533 {
 534         kmem_cache_free(iommu_iova_cache, iova);
 535 }
 536
 537
 538 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 539 {
 540         unsigned long sagaw;
 541         int agaw = -1;
 542
 543         sagaw = cap_sagaw(iommu->cap);
 544         for (agaw = width_to_agaw(max_gaw);
 545              agaw >= 0; agaw--) {
 546                 if (test_bit(agaw, &sagaw))
 547                         break;
 548         }
 549
 550         return agaw;
 551 }
 552
 553 /*
 554  * Calculate max SAGAW for each iommu.
 555  */
 556 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 557 {
 558         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 559 }
 560
 561 /*
 562  * calculate agaw for each iommu.
 563  * "SAGAW" may be different across iommus, use a default agaw, and
 564  * get a supported less agaw for iommus that don't support the default agaw.
 565  */
 566 int iommu_calculate_agaw(struct intel_iommu *iommu)
 567 {
 568         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 569 }
 570
 571 /* This functionin only returns single iommu in a domain */
 572 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 573 {
 574         int iommu_id;
 575
 576         /* si_domain and vm domain should not get here. */
 577         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 578         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 579
 580         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 581         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 582                 return NULL;
 583
 584         return g_iommus[iommu_id];
 585 }
 586
 587 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 588 {
 589         int i;
 590
 591         domain->iommu_coherency = 1;
 592
 593         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 594                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 595                         domain->iommu_coherency = 0;
 596                         break;
 597                 }
 598         }
 599 }
 600
 601 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 602 {
 603         int i;
 604
 605         domain->iommu_snooping = 1;
 606
 607         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 608                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 609                         domain->iommu_snooping = 0;
 610                         break;
 611                 }
 612         }
 613 }
 614
 615 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 616 {
 617         struct dmar_drhd_unit *drhd;
 618         struct intel_iommu *iommu = NULL;
 619         int mask = 0xf;
 620
 621         if (!intel_iommu_superpage) {
 622                 domain->iommu_superpage = 0;
 623                 return;
 624         }
 625
 626         /* set iommu_superpage to the smallest common denominator */
 627         for_each_active_iommu(iommu, drhd) {
 628                 mask &= cap_super_page_val(iommu->cap);
 629                 if (!mask) {
 630                         break;
 631                 }
 632         }
 633         domain->iommu_superpage = fls(mask);
 634 }
 635
 636 /* Some capabilities may be different across iommus */
 637 static void domain_update_iommu_cap(struct dmar_domain *domain)
 638 {
 639         domain_update_iommu_coherency(domain);
 640         domain_update_iommu_snooping(domain);
 641         domain_update_iommu_superpage(domain);
 642 }
 643
 644 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 645 {
 646         struct dmar_drhd_unit *drhd = NULL;
 647         int i;
 648
 649         for_each_drhd_unit(drhd) {
 650                 if (drhd->ignored)
 651                         continue;
 652                 if (segment != drhd->segment)
 653                         continue;
 654
 655                 for (i = 0; i < drhd->devices_cnt; i++) {
 656                         if (drhd->devices[i] &&
 657                             drhd->devices[i]->bus->number == bus &&
 658                             drhd->devices[i]->devfn == devfn)
 659                                 return drhd->iommu;
 660                         if (drhd->devices[i] &&
 661                             drhd->devices[i]->subordinate &&
 662                             drhd->devices[i]->subordinate->number <= bus &&
 663                             drhd->devices[i]->subordinate->subordinate >= bus)
 664                                 return drhd->iommu;
 665                 }
 666
 667                 if (drhd->include_all)
 668                         return drhd->iommu;
 669         }
 670
 671         return NULL;
 672 }
 673
 674 static void domain_flush_cache(struct dmar_domain *domain,
 675                                void *addr, int size)
 676 {
 677         if (!domain->iommu_coherency)
 678                 clflush_cache_range(addr, size);
 679 }
 680
 681 /* Gets context entry for a given bus and devfn */
 682 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 683                 u8 bus, u8 devfn)
 684 {
 685         struct root_entry *root;
 686         struct context_entry *context;
 687         unsigned long phy_addr;
 688         unsigned long flags;
 689
 690         spin_lock_irqsave(&iommu->lock, flags);
 691         root = &iommu->root_entry[bus];
 692         context = get_context_addr_from_root(root);
 693         if (!context) {
 694                 context = (struct context_entry *)
 695                                 alloc_pgtable_page(iommu->node);
 696                 if (!context) {
 697                         spin_unlock_irqrestore(&iommu->lock, flags);
 698                         return NULL;
 699                 }
 700                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 701                 phy_addr = virt_to_phys((void *)context);
 702                 set_root_value(root, phy_addr);
 703                 set_root_present(root);
 704                 __iommu_flush_cache(iommu, root, sizeof(*root));
 705         }
 706         spin_unlock_irqrestore(&iommu->lock, flags);
 707         return &context[devfn];
 708 }
 709
 710 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 711 {
 712         struct root_entry *root;
 713         struct context_entry *context;
 714         int ret;
 715         unsigned long flags;
 716
 717         spin_lock_irqsave(&iommu->lock, flags);
 718         root = &iommu->root_entry[bus];
 719         context = get_context_addr_from_root(root);
 720         if (!context) {
 721                 ret = 0;
 722                 goto out;
 723         }
 724         ret = context_present(&context[devfn]);
 725 out:
 726         spin_unlock_irqrestore(&iommu->lock, flags);
 727         return ret;
 728 }
 729
 730 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 731 {
 732         struct root_entry *root;
 733         struct context_entry *context;
 734         unsigned long flags;
 735
 736         spin_lock_irqsave(&iommu->lock, flags);
 737         root = &iommu->root_entry[bus];
 738         context = get_context_addr_from_root(root);
 739         if (context) {
 740                 context_clear_entry(&context[devfn]);
 741                 __iommu_flush_cache(iommu, &context[devfn], \
 742                         sizeof(*context));
 743         }
 744         spin_unlock_irqrestore(&iommu->lock, flags);
 745 }
 746
 747 static void free_context_table(struct intel_iommu *iommu)
 748 {
 749         struct root_entry *root;
 750         int i;
 751         unsigned long flags;
 752         struct context_entry *context;
 753
 754         spin_lock_irqsave(&iommu->lock, flags);
 755         if (!iommu->root_entry) {
 756                 goto out;
 757         }
 758         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 759                 root = &iommu->root_entry[i];
 760                 context = get_context_addr_from_root(root);
 761                 if (context)
 762                         free_pgtable_page(context);
 763         }
 764         free_pgtable_page(iommu->root_entry);
 765         iommu->root_entry = NULL;
 766 out:
 767         spin_unlock_irqrestore(&iommu->lock, flags);
 768 }
 769
 770 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 771                                       unsigned long pfn, int target_level)
 772 {
 773         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 774         struct dma_pte *parent, *pte = NULL;
 775         int level = agaw_to_level(domain->agaw);
 776         int offset;
 777
 778         BUG_ON(!domain->pgd);
 779         BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 780         parent = domain->pgd;
 781
 782         while (level > 0) {
 783                 void *tmp_page;
 784
 785                 offset = pfn_level_offset(pfn, level);
 786                 pte = &parent[offset];
 787                 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 788                         break;
 789                 if (level == target_level)
 790                         break;
 791
 792                 if (!dma_pte_present(pte)) {
 793                         uint64_t pteval;
 794
 795                         tmp_page = alloc_pgtable_page(domain->nid);
 796
 797                         if (!tmp_page)
 798                                 return NULL;
 799
 800                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 801                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 802                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 803                                 /* Someone else set it while we were thinking; use theirs. */
 804                                 free_pgtable_page(tmp_page);
 805                         } else {
 806                                 dma_pte_addr(pte);
 807                                 domain_flush_cache(domain, pte, sizeof(*pte));
 808                         }
 809                 }
 810                 parent = phys_to_virt(dma_pte_addr(pte));
 811                 level--;
 812         }
 813
 814         return pte;
 815 }
 816
 817
 818 /* return address's pte at specific level */
 819 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 820                                          unsigned long pfn,
 821                                          int level, int *large_page)
 822 {
 823         struct dma_pte *parent, *pte = NULL;
 824         int total = agaw_to_level(domain->agaw);
 825         int offset;
 826
 827         parent = domain->pgd;
 828         while (level <= total) {
 829                 offset = pfn_level_offset(pfn, total);
 830                 pte = &parent[offset];
 831                 if (level == total)
 832                         return pte;
 833
 834                 if (!dma_pte_present(pte)) {
 835                         *large_page = total;
 836                         break;
 837                 }
 838
 839                 if (pte->val & DMA_PTE_LARGE_PAGE) {
 840                         *large_page = total;
 841                         return pte;
 842                 }
 843
 844                 parent = phys_to_virt(dma_pte_addr(pte));
 845                 total--;
 846         }
 847         return NULL;
 848 }
 849
 850 /* clear last level pte, a tlb flush should be followed */
 851 static int dma_pte_clear_range(struct dmar_domain *domain,
 852                                 unsigned long start_pfn,
 853                                 unsigned long last_pfn)
 854 {
 855         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 856         unsigned int large_page = 1;
 857         struct dma_pte *first_pte, *pte;
 858         int order;
 859
 860         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 861         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 862         BUG_ON(start_pfn > last_pfn);
 863
 864         /* we don't need lock here; nobody else touches the iova range */
 865         do {
 866                 large_page = 1;
 867                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 868                 if (!pte) {
 869                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 870                         continue;
 871                 }
 872                 do {
 873                         dma_clear_pte(pte);
 874                         start_pfn += lvl_to_nr_pages(large_page);
 875                         pte++;
 876                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 877
 878                 domain_flush_cache(domain, first_pte,
 879                                    (void *)pte - (void *)first_pte);
 880
 881         } while (start_pfn && start_pfn <= last_pfn);
 882
 883         order = (large_page - 1) * 9;
 884         return order;
 885 }
 886
 887 /* free page table pages. last level pte should already be cleared */
 888 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 889                                    unsigned long start_pfn,
 890                                    unsigned long last_pfn)
 891 {
 892         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 893         struct dma_pte *first_pte, *pte;
 894         int total = agaw_to_level(domain->agaw);
 895         int level;
 896         unsigned long tmp;
 897         int large_page = 2;
 898
 899         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 900         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 901         BUG_ON(start_pfn > last_pfn);
 902
 903         /* We don't need lock here; nobody else touches the iova range */
 904         level = 2;
 905         while (level <= total) {
 906                 tmp = align_to_level(start_pfn, level);
 907
 908                 /* If we can't even clear one PTE at this level, we're done */
 909                 if (tmp + level_size(level) - 1 > last_pfn)
 910                         return;
 911
 912                 do {
 913                         large_page = level;
 914                         first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
 915                         if (large_page > level)
 916                                 level = large_page + 1;
 917                         if (!pte) {
 918                                 tmp = align_to_level(tmp + 1, level + 1);
 919                                 continue;
 920                         }
 921                         do {
 922                                 if (dma_pte_present(pte)) {
 923                                         free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
 924                                         dma_clear_pte(pte);
 925                                 }
 926                                 pte++;
 927                                 tmp += level_size(level);
 928                         } while (!first_pte_in_page(pte) &&
 929                                  tmp + level_size(level) - 1 <= last_pfn);
 930
 931                         domain_flush_cache(domain, first_pte,
 932                                            (void *)pte - (void *)first_pte);
 933
 934                 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
 935                 level++;
 936         }
 937         /* free pgd */
 938         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 939                 free_pgtable_page(domain->pgd);
 940                 domain->pgd = NULL;
 941         }
 942 }
 943
 944 /* iommu handling */
 945 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 946 {
 947         struct root_entry *root;
 948         unsigned long flags;
 949
 950         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 951         if (!root)
 952                 return -ENOMEM;
 953
 954         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 955
 956         spin_lock_irqsave(&iommu->lock, flags);
 957         iommu->root_entry = root;
 958         spin_unlock_irqrestore(&iommu->lock, flags);
 959
 960         return 0;
 961 }
 962
 963 static void iommu_set_root_entry(struct intel_iommu *iommu)
 964 {
 965         void *addr;
 966         u32 sts;
 967         unsigned long flag;
 968
 969         addr = iommu->root_entry;
 970
 971         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 972         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 973
 974         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 975
 976         /* Make sure hardware complete it */
 977         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 978                       readl, (sts & DMA_GSTS_RTPS), sts);
 979
 980         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 981 }
 982
 983 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 984 {
 985         u32 val;
 986         unsigned long flag;
 987
 988         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 989                 return;
 990
 991         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 992         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 993
 994         /* Make sure hardware complete it */
 995         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 996                       readl, (!(val & DMA_GSTS_WBFS)), val);
 997
 998         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 999 }
1000
1001 /* return value determine if we need a write buffer flush */
1002 static void __iommu_flush_context(struct intel_iommu *iommu,
1003                                   u16 did, u16 source_id, u8 function_mask,
1004                                   u64 type)
1005 {
1006         u64 val = 0;
1007         unsigned long flag;
1008
1009         switch (type) {
1010         case DMA_CCMD_GLOBAL_INVL:
1011                 val = DMA_CCMD_GLOBAL_INVL;
1012                 break;
1013         case DMA_CCMD_DOMAIN_INVL:
1014                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1015                 break;
1016         case DMA_CCMD_DEVICE_INVL:
1017                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1018                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1019                 break;
1020         default:
1021                 BUG();
1022         }
1023         val |= DMA_CCMD_ICC;
1024
1025         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1026         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1027
1028         /* Make sure hardware complete it */
1029         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1030                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1031
1032         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1033 }
1034
1035 /* return value determine if we need a write buffer flush */
1036 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1037                                 u64 addr, unsigned int size_order, u64 type)
1038 {
1039         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1040         u64 val = 0, val_iva = 0;
1041         unsigned long flag;
1042
1043         switch (type) {
1044         case DMA_TLB_GLOBAL_FLUSH:
1045                 /* global flush doesn't need set IVA_REG */
1046                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1047                 break;
1048         case DMA_TLB_DSI_FLUSH:
1049                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1050                 break;
1051         case DMA_TLB_PSI_FLUSH:
1052                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1053                 /* Note: always flush non-leaf currently */
1054                 val_iva = size_order | addr;
1055                 break;
1056         default:
1057                 BUG();
1058         }
1059         /* Note: set drain read/write */
1060 #if 0
1061         /*
1062          * This is probably to be super secure.. Looks like we can
1063          * ignore it without any impact.
1064          */
1065         if (cap_read_drain(iommu->cap))
1066                 val |= DMA_TLB_READ_DRAIN;
1067 #endif
1068         if (cap_write_drain(iommu->cap))
1069                 val |= DMA_TLB_WRITE_DRAIN;
1070
1071         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1072         /* Note: Only uses first TLB reg currently */
1073         if (val_iva)
1074                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1075         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1076
1077         /* Make sure hardware complete it */
1078         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1079                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1080
1081         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1082
1083         /* check IOTLB invalidation granularity */
1084         if (DMA_TLB_IAIG(val) == 0)
1085                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1086         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1087                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1088                         (unsigned long long)DMA_TLB_IIRG(type),
1089                         (unsigned long long)DMA_TLB_IAIG(val));
1090 }
1091
1092 static struct device_domain_info *iommu_support_dev_iotlb(
1093         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1094 {
1095         int found = 0;
1096         unsigned long flags;
1097         struct device_domain_info *info;
1098         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1099
1100         if (!ecap_dev_iotlb_support(iommu->ecap))
1101                 return NULL;
1102
1103         if (!iommu->qi)
1104                 return NULL;
1105
1106         spin_lock_irqsave(&device_domain_lock, flags);
1107         list_for_each_entry(info, &domain->devices, link)
1108                 if (info->bus == bus && info->devfn == devfn) {
1109                         found = 1;
1110                         break;
1111                 }
1112         spin_unlock_irqrestore(&device_domain_lock, flags);
1113
1114         if (!found || !info->dev)
1115                 return NULL;
1116
1117         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1118                 return NULL;
1119
1120         if (!dmar_find_matched_atsr_unit(info->dev))
1121                 return NULL;
1122
1123         info->iommu = iommu;
1124
1125         return info;
1126 }
1127
1128 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1129 {
1130         if (!info)
1131                 return;
1132
1133         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1134 }
1135
1136 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1137 {
1138         if (!info->dev || !pci_ats_enabled(info->dev))
1139                 return;
1140
1141         pci_disable_ats(info->dev);
1142 }
1143
1144 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1145                                   u64 addr, unsigned mask)
1146 {
1147         u16 sid, qdep;
1148         unsigned long flags;
1149         struct device_domain_info *info;
1150
1151         spin_lock_irqsave(&device_domain_lock, flags);
1152         list_for_each_entry(info, &domain->devices, link) {
1153                 if (!info->dev || !pci_ats_enabled(info->dev))
1154                         continue;
1155
1156                 sid = info->bus << 8 | info->devfn;
1157                 qdep = pci_ats_queue_depth(info->dev);
1158                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1159         }
1160         spin_unlock_irqrestore(&device_domain_lock, flags);
1161 }
1162
1163 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1164                                   unsigned long pfn, unsigned int pages, int map)
1165 {
1166         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1167         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1168
1169         BUG_ON(pages == 0);
1170
1171         /*
1172          * Fallback to domain selective flush if no PSI support or the size is
1173          * too big.
1174          * PSI requires page size to be 2 ^ x, and the base address is naturally
1175          * aligned to the size
1176          */
1177         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1178                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1179                                                 DMA_TLB_DSI_FLUSH);
1180         else
1181                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1182                                                 DMA_TLB_PSI_FLUSH);
1183
1184         /*
1185          * In caching mode, changes of pages from non-present to present require
1186          * flush. However, device IOTLB doesn't need to be flushed in this case.
1187          */
1188         if (!cap_caching_mode(iommu->cap) || !map)
1189                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1190 }
1191
1192 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1193 {
1194         u32 pmen;
1195         unsigned long flags;
1196
1197         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1198         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1199         pmen &= ~DMA_PMEN_EPM;
1200         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1201
1202         /* wait for the protected region status bit to clear */
1203         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1204                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1205
1206         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1207 }
1208
1209 static int iommu_enable_translation(struct intel_iommu *iommu)
1210 {
1211         u32 sts;
1212         unsigned long flags;
1213
1214         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1215         iommu->gcmd |= DMA_GCMD_TE;
1216         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1217
1218         /* Make sure hardware complete it */
1219         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1220                       readl, (sts & DMA_GSTS_TES), sts);
1221
1222         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1223         return 0;
1224 }
1225
1226 static int iommu_disable_translation(struct intel_iommu *iommu)
1227 {
1228         u32 sts;
1229         unsigned long flag;
1230
1231         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1232         iommu->gcmd &= ~DMA_GCMD_TE;
1233         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1234
1235         /* Make sure hardware complete it */
1236         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1237                       readl, (!(sts & DMA_GSTS_TES)), sts);
1238
1239         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1240         return 0;
1241 }
1242
1243
1244 static int iommu_init_domains(struct intel_iommu *iommu)
1245 {
1246         unsigned long ndomains;
1247         unsigned long nlongs;
1248
1249         ndomains = cap_ndoms(iommu->cap);
1250         pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
1251                         ndomains);
1252         nlongs = BITS_TO_LONGS(ndomains);
1253
1254         spin_lock_init(&iommu->lock);
1255
1256         /* TBD: there might be 64K domains,
1257          * consider other allocation for future chip
1258          */
1259         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1260         if (!iommu->domain_ids) {
1261                 printk(KERN_ERR "Allocating domain id array failed\n");
1262                 return -ENOMEM;
1263         }
1264         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1265                         GFP_KERNEL);
1266         if (!iommu->domains) {
1267                 printk(KERN_ERR "Allocating domain array failed\n");
1268                 return -ENOMEM;
1269         }
1270
1271         /*
1272          * if Caching mode is set, then invalid translations are tagged
1273          * with domainid 0. Hence we need to pre-allocate it.
1274          */
1275         if (cap_caching_mode(iommu->cap))
1276                 set_bit(0, iommu->domain_ids);
1277         return 0;
1278 }
1279
1280
1281 static void domain_exit(struct dmar_domain *domain);
1282 static void vm_domain_exit(struct dmar_domain *domain);
1283
1284 void free_dmar_iommu(struct intel_iommu *iommu)
1285 {
1286         struct dmar_domain *domain;
1287         int i;
1288         unsigned long flags;
1289
1290         if ((iommu->domains) && (iommu->domain_ids)) {
1291                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1292                         domain = iommu->domains[i];
1293                         clear_bit(i, iommu->domain_ids);
1294
1295                         spin_lock_irqsave(&domain->iommu_lock, flags);
1296                         if (--domain->iommu_count == 0) {
1297                                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1298                                         vm_domain_exit(domain);
1299                                 else
1300                                         domain_exit(domain);
1301                         }
1302                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1303                 }
1304         }
1305
1306         if (iommu->gcmd & DMA_GCMD_TE)
1307                 iommu_disable_translation(iommu);
1308
1309         if (iommu->irq) {
1310                 irq_set_handler_data(iommu->irq, NULL);
1311                 /* This will mask the irq */
1312                 free_irq(iommu->irq, iommu);
1313                 destroy_irq(iommu->irq);
1314         }
1315
1316         kfree(iommu->domains);
1317         kfree(iommu->domain_ids);
1318
1319         g_iommus[iommu->seq_id] = NULL;
1320
1321         /* if all iommus are freed, free g_iommus */
1322         for (i = 0; i < g_num_of_iommus; i++) {
1323                 if (g_iommus[i])
1324                         break;
1325         }
1326
1327         if (i == g_num_of_iommus)
1328                 kfree(g_iommus);
1329
1330         /* free context mapping */
1331         free_context_table(iommu);
1332 }
1333
1334 static struct dmar_domain *alloc_domain(void)
1335 {
1336         struct dmar_domain *domain;
1337
1338         domain = alloc_domain_mem();
1339         if (!domain)
1340                 return NULL;
1341
1342         domain->nid = -1;
1343         memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1344         domain->flags = 0;
1345
1346         return domain;
1347 }
1348
1349 static int iommu_attach_domain(struct dmar_domain *domain,
1350                                struct intel_iommu *iommu)
1351 {
1352         int num;
1353         unsigned long ndomains;
1354         unsigned long flags;
1355
1356         ndomains = cap_ndoms(iommu->cap);
1357
1358         spin_lock_irqsave(&iommu->lock, flags);
1359
1360         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1361         if (num >= ndomains) {
1362                 spin_unlock_irqrestore(&iommu->lock, flags);
1363                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1364                 return -ENOMEM;
1365         }
1366
1367         domain->id = num;
1368         set_bit(num, iommu->domain_ids);
1369         set_bit(iommu->seq_id, domain->iommu_bmp);
1370         iommu->domains[num] = domain;
1371         spin_unlock_irqrestore(&iommu->lock, flags);
1372
1373         return 0;
1374 }
1375
1376 static void iommu_detach_domain(struct dmar_domain *domain,
1377                                 struct intel_iommu *iommu)
1378 {
1379         unsigned long flags;
1380         int num, ndomains;
1381         int found = 0;
1382
1383         spin_lock_irqsave(&iommu->lock, flags);
1384         ndomains = cap_ndoms(iommu->cap);
1385         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1386                 if (iommu->domains[num] == domain) {
1387                         found = 1;
1388                         break;
1389                 }
1390         }
1391
1392         if (found) {
1393                 clear_bit(num, iommu->domain_ids);
1394                 clear_bit(iommu->seq_id, domain->iommu_bmp);
1395                 iommu->domains[num] = NULL;
1396         }
1397         spin_unlock_irqrestore(&iommu->lock, flags);
1398 }
1399
1400 static struct iova_domain reserved_iova_list;
1401 static struct lock_class_key reserved_rbtree_key;
1402
1403 static int dmar_init_reserved_ranges(void)
1404 {
1405         struct pci_dev *pdev = NULL;
1406         struct iova *iova;
1407         int i;
1408
1409         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1410
1411         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1412                 &reserved_rbtree_key);
1413
1414         /* IOAPIC ranges shouldn't be accessed by DMA */
1415         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1416                 IOVA_PFN(IOAPIC_RANGE_END));
1417         if (!iova) {
1418                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1419                 return -ENODEV;
1420         }
1421
1422         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1423         for_each_pci_dev(pdev) {
1424                 struct resource *r;
1425
1426                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1427                         r = &pdev->resource[i];
1428                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1429                                 continue;
1430                         iova = reserve_iova(&reserved_iova_list,
1431                                             IOVA_PFN(r->start),
1432                                             IOVA_PFN(r->end));
1433                         if (!iova) {
1434                                 printk(KERN_ERR "Reserve iova failed\n");
1435                                 return -ENODEV;
1436                         }
1437                 }
1438         }
1439         return 0;
1440 }
1441
1442 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1443 {
1444         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1445 }
1446
1447 static inline int guestwidth_to_adjustwidth(int gaw)
1448 {
1449         int agaw;
1450         int r = (gaw - 12) % 9;
1451
1452         if (r == 0)
1453                 agaw = gaw;
1454         else
1455                 agaw = gaw + 9 - r;
1456         if (agaw > 64)
1457                 agaw = 64;
1458         return agaw;
1459 }
1460
1461 static int domain_init(struct dmar_domain *domain, int guest_width)
1462 {
1463         struct intel_iommu *iommu;
1464         int adjust_width, agaw;
1465         unsigned long sagaw;
1466
1467         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1468         spin_lock_init(&domain->iommu_lock);
1469
1470         domain_reserve_special_ranges(domain);
1471
1472         /* calculate AGAW */
1473         iommu = domain_get_iommu(domain);
1474         if (guest_width > cap_mgaw(iommu->cap))
1475                 guest_width = cap_mgaw(iommu->cap);
1476         domain->gaw = guest_width;
1477         adjust_width = guestwidth_to_adjustwidth(guest_width);
1478         agaw = width_to_agaw(adjust_width);
1479         sagaw = cap_sagaw(iommu->cap);
1480         if (!test_bit(agaw, &sagaw)) {
1481                 /* hardware doesn't support it, choose a bigger one */
1482                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1483                 agaw = find_next_bit(&sagaw, 5, agaw);
1484                 if (agaw >= 5)
1485                         return -ENODEV;
1486         }
1487         domain->agaw = agaw;
1488         INIT_LIST_HEAD(&domain->devices);
1489
1490         if (ecap_coherent(iommu->ecap))
1491                 domain->iommu_coherency = 1;
1492         else
1493                 domain->iommu_coherency = 0;
1494
1495         if (ecap_sc_support(iommu->ecap))
1496                 domain->iommu_snooping = 1;
1497         else
1498                 domain->iommu_snooping = 0;
1499
1500         domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1501         domain->iommu_count = 1;
1502         domain->nid = iommu->node;
1503
1504         /* always allocate the top pgd */
1505         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1506         if (!domain->pgd)
1507                 return -ENOMEM;
1508         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1509         return 0;
1510 }
1511
1512 static void domain_exit(struct dmar_domain *domain)
1513 {
1514         struct dmar_drhd_unit *drhd;
1515         struct intel_iommu *iommu;
1516
1517         /* Domain 0 is reserved, so dont process it */
1518         if (!domain)
1519                 return;
1520
1521         /* Flush any lazy unmaps that may reference this domain */
1522         if (!intel_iommu_strict)
1523                 flush_unmaps_timeout(0);
1524
1525         domain_remove_dev_info(domain);
1526         /* destroy iovas */
1527         put_iova_domain(&domain->iovad);
1528
1529         /* clear ptes */
1530         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1531
1532         /* free page tables */
1533         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1534
1535         for_each_active_iommu(iommu, drhd)
1536                 if (test_bit(iommu->seq_id, domain->iommu_bmp))
1537                         iommu_detach_domain(domain, iommu);
1538
1539         free_domain_mem(domain);
1540 }
1541
1542 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1543                                  u8 bus, u8 devfn, int translation)
1544 {
1545         struct context_entry *context;
1546         unsigned long flags;
1547         struct intel_iommu *iommu;
1548         struct dma_pte *pgd;
1549         unsigned long num;
1550         unsigned long ndomains;
1551         int id;
1552         int agaw;
1553         struct device_domain_info *info = NULL;
1554
1555         pr_debug("Set context mapping for %02x:%02x.%d\n",
1556                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1557
1558         BUG_ON(!domain->pgd);
1559         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1560                translation != CONTEXT_TT_MULTI_LEVEL);
1561
1562         iommu = device_to_iommu(segment, bus, devfn);
1563         if (!iommu)
1564                 return -ENODEV;
1565
1566         context = device_to_context_entry(iommu, bus, devfn);
1567         if (!context)
1568                 return -ENOMEM;
1569         spin_lock_irqsave(&iommu->lock, flags);
1570         if (context_present(context)) {
1571                 spin_unlock_irqrestore(&iommu->lock, flags);
1572                 return 0;
1573         }
1574
1575         id = domain->id;
1576         pgd = domain->pgd;
1577
1578         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1579             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1580                 int found = 0;
1581
1582                 /* find an available domain id for this device in iommu */
1583                 ndomains = cap_ndoms(iommu->cap);
1584                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1585                         if (iommu->domains[num] == domain) {
1586                                 id = num;
1587                                 found = 1;
1588                                 break;
1589                         }
1590                 }
1591
1592                 if (found == 0) {
1593                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1594                         if (num >= ndomains) {
1595                                 spin_unlock_irqrestore(&iommu->lock, flags);
1596                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1597                                 return -EFAULT;
1598                         }
1599
1600                         set_bit(num, iommu->domain_ids);
1601                         iommu->domains[num] = domain;
1602                         id = num;
1603                 }
1604
1605                 /* Skip top levels of page tables for
1606                  * iommu which has less agaw than default.
1607                  * Unnecessary for PT mode.
1608                  */
1609                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1610                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1611                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1612                                 if (!dma_pte_present(pgd)) {
1613                                         spin_unlock_irqrestore(&iommu->lock, flags);
1614                                         return -ENOMEM;
1615                                 }
1616                         }
1617                 }
1618         }
1619
1620         context_set_domain_id(context, id);
1621
1622         if (translation != CONTEXT_TT_PASS_THROUGH) {
1623                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1624                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1625                                      CONTEXT_TT_MULTI_LEVEL;
1626         }
1627         /*
1628          * In pass through mode, AW must be programmed to indicate the largest
1629          * AGAW value supported by hardware. And ASR is ignored by hardware.
1630          */
1631         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1632                 context_set_address_width(context, iommu->msagaw);
1633         else {
1634                 context_set_address_root(context, virt_to_phys(pgd));
1635                 context_set_address_width(context, iommu->agaw);
1636         }
1637
1638         context_set_translation_type(context, translation);
1639         context_set_fault_enable(context);
1640         context_set_present(context);
1641         domain_flush_cache(domain, context, sizeof(*context));
1642
1643         /*
1644          * It's a non-present to present mapping. If hardware doesn't cache
1645          * non-present entry we only need to flush the write-buffer. If the
1646          * _does_ cache non-present entries, then it does so in the special
1647          * domain #0, which we have to flush:
1648          */
1649         if (cap_caching_mode(iommu->cap)) {
1650                 iommu->flush.flush_context(iommu, 0,
1651                                            (((u16)bus) << 8) | devfn,
1652                                            DMA_CCMD_MASK_NOBIT,
1653                                            DMA_CCMD_DEVICE_INVL);
1654                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1655         } else {
1656                 iommu_flush_write_buffer(iommu);
1657         }
1658         iommu_enable_dev_iotlb(info);
1659         spin_unlock_irqrestore(&iommu->lock, flags);
1660
1661         spin_lock_irqsave(&domain->iommu_lock, flags);
1662         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1663                 domain->iommu_count++;
1664                 if (domain->iommu_count == 1)
1665                         domain->nid = iommu->node;
1666                 domain_update_iommu_cap(domain);
1667         }
1668         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1669         return 0;
1670 }
1671
1672 static int
1673 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1674                         int translation)
1675 {
1676         int ret;
1677         struct pci_dev *tmp, *parent;
1678
1679         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1680                                          pdev->bus->number, pdev->devfn,
1681                                          translation);
1682         if (ret)
1683                 return ret;
1684
1685         /* dependent device mapping */
1686         tmp = pci_find_upstream_pcie_bridge(pdev);
1687         if (!tmp)
1688                 return 0;
1689         /* Secondary interface's bus number and devfn 0 */
1690         parent = pdev->bus->self;
1691         while (parent != tmp) {
1692                 ret = domain_context_mapping_one(domain,
1693                                                  pci_domain_nr(parent->bus),
1694                                                  parent->bus->number,
1695                                                  parent->devfn, translation);
1696                 if (ret)
1697                         return ret;
1698                 parent = parent->bus->self;
1699         }
1700         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1701                 return domain_context_mapping_one(domain,
1702                                         pci_domain_nr(tmp->subordinate),
1703                                         tmp->subordinate->number, 0,
1704                                         translation);
1705         else /* this is a legacy PCI bridge */
1706                 return domain_context_mapping_one(domain,
1707                                                   pci_domain_nr(tmp->bus),
1708                                                   tmp->bus->number,
1709                                                   tmp->devfn,
1710                                                   translation);
1711 }
1712
1713 static int domain_context_mapped(struct pci_dev *pdev)
1714 {
1715         int ret;
1716         struct pci_dev *tmp, *parent;
1717         struct intel_iommu *iommu;
1718
1719         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1720                                 pdev->devfn);
1721         if (!iommu)
1722                 return -ENODEV;
1723
1724         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1725         if (!ret)
1726                 return ret;
1727         /* dependent device mapping */
1728         tmp = pci_find_upstream_pcie_bridge(pdev);
1729         if (!tmp)
1730                 return ret;
1731         /* Secondary interface's bus number and devfn 0 */
1732         parent = pdev->bus->self;
1733         while (parent != tmp) {
1734                 ret = device_context_mapped(iommu, parent->bus->number,
1735                                             parent->devfn);
1736                 if (!ret)
1737                         return ret;
1738                 parent = parent->bus->self;
1739         }
1740         if (pci_is_pcie(tmp))
1741                 return device_context_mapped(iommu, tmp->subordinate->number,
1742                                              0);
1743         else
1744                 return device_context_mapped(iommu, tmp->bus->number,
1745                                              tmp->devfn);
1746 }
1747
1748 /* Returns a number of VTD pages, but aligned to MM page size */
1749 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1750                                             size_t size)
1751 {
1752         host_addr &= ~PAGE_MASK;
1753         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1754 }
1755
1756 /* Return largest possible superpage level for a given mapping */
1757 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1758                                           unsigned long iov_pfn,
1759                                           unsigned long phy_pfn,
1760                                           unsigned long pages)
1761 {
1762         int support, level = 1;
1763         unsigned long pfnmerge;
1764
1765         support = domain->iommu_superpage;
1766
1767         /* To use a large page, the virtual *and* physical addresses
1768            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1769            of them will mean we have to use smaller pages. So just
1770            merge them and check both at once. */
1771         pfnmerge = iov_pfn | phy_pfn;
1772
1773         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1774                 pages >>= VTD_STRIDE_SHIFT;
1775                 if (!pages)
1776                         break;
1777                 pfnmerge >>= VTD_STRIDE_SHIFT;
1778                 level++;
1779                 support--;
1780         }
1781         return level;
1782 }
1783
1784 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1785                             struct scatterlist *sg, unsigned long phys_pfn,
1786                             unsigned long nr_pages, int prot)
1787 {
1788         struct dma_pte *first_pte = NULL, *pte = NULL;
1789         phys_addr_t uninitialized_var(pteval);
1790         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1791         unsigned long sg_res;
1792         unsigned int largepage_lvl = 0;
1793         unsigned long lvl_pages = 0;
1794
1795         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1796
1797         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1798                 return -EINVAL;
1799
1800         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1801
1802         if (sg)
1803                 sg_res = 0;
1804         else {
1805                 sg_res = nr_pages + 1;
1806                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1807         }
1808
1809         while (nr_pages > 0) {
1810                 uint64_t tmp;
1811
1812                 if (!sg_res) {
1813                         sg_res = aligned_nrpages(sg->offset, sg->length);
1814                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1815                         sg->dma_length = sg->length;
1816                         pteval = page_to_phys(sg_page(sg)) | prot;
1817                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1818                 }
1819
1820                 if (!pte) {
1821                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1822
1823                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1824                         if (!pte)
1825                                 return -ENOMEM;
1826                         /* It is large page*/
1827                         if (largepage_lvl > 1)
1828                                 pteval |= DMA_PTE_LARGE_PAGE;
1829                         else
1830                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1831
1832                 }
1833                 /* We don't need lock here, nobody else
1834                  * touches the iova range
1835                  */
1836                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1837                 if (tmp) {
1838                         static int dumps = 5;
1839                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1840                                iov_pfn, tmp, (unsigned long long)pteval);
1841                         if (dumps) {
1842                                 dumps--;
1843                                 debug_dma_dump_mappings(NULL);
1844                         }
1845                         WARN_ON(1);
1846                 }
1847
1848                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1849
1850                 BUG_ON(nr_pages < lvl_pages);
1851                 BUG_ON(sg_res < lvl_pages);
1852
1853                 nr_pages -= lvl_pages;
1854                 iov_pfn += lvl_pages;
1855                 phys_pfn += lvl_pages;
1856                 pteval += lvl_pages * VTD_PAGE_SIZE;
1857                 sg_res -= lvl_pages;
1858
1859                 /* If the next PTE would be the first in a new page, then we
1860                    need to flush the cache on the entries we've just written.
1861                    And then we'll need to recalculate 'pte', so clear it and
1862                    let it get set again in the if (!pte) block above.
1863
1864                    If we're done (!nr_pages) we need to flush the cache too.
1865
1866                    Also if we've been setting superpages, we may need to
1867                    recalculate 'pte' and switch back to smaller pages for the
1868                    end of the mapping, if the trailing size is not enough to
1869                    use another superpage (i.e. sg_res < lvl_pages). */
1870                 pte++;
1871                 if (!nr_pages || first_pte_in_page(pte) ||
1872                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
1873                         domain_flush_cache(domain, first_pte,
1874                                            (void *)pte - (void *)first_pte);
1875                         pte = NULL;
1876                 }
1877
1878                 if (!sg_res && nr_pages)
1879                         sg = sg_next(sg);
1880         }
1881         return 0;
1882 }
1883
1884 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1885                                     struct scatterlist *sg, unsigned long nr_pages,
1886                                     int prot)
1887 {
1888         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1889 }
1890
1891 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1892                                      unsigned long phys_pfn, unsigned long nr_pages,
1893                                      int prot)
1894 {
1895         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1896 }
1897
1898 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1899 {
1900         if (!iommu)
1901                 return;
1902
1903         clear_context_table(iommu, bus, devfn);
1904         iommu->flush.flush_context(iommu, 0, 0, 0,
1905                                            DMA_CCMD_GLOBAL_INVL);
1906         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1907 }
1908
1909 static void domain_remove_dev_info(struct dmar_domain *domain)
1910 {
1911         struct device_domain_info *info;
1912         unsigned long flags;
1913         struct intel_iommu *iommu;
1914
1915         spin_lock_irqsave(&device_domain_lock, flags);
1916         while (!list_empty(&domain->devices)) {
1917                 info = list_entry(domain->devices.next,
1918                         struct device_domain_info, link);
1919                 list_del(&info->link);
1920                 list_del(&info->global);
1921                 if (info->dev)
1922                         info->dev->dev.archdata.iommu = NULL;
1923                 spin_unlock_irqrestore(&device_domain_lock, flags);
1924
1925                 iommu_disable_dev_iotlb(info);
1926                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1927                 iommu_detach_dev(iommu, info->bus, info->devfn);
1928                 free_devinfo_mem(info);
1929
1930                 spin_lock_irqsave(&device_domain_lock, flags);
1931         }
1932         spin_unlock_irqrestore(&device_domain_lock, flags);
1933 }
1934
1935 /*
1936  * find_domain
1937  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1938  */
1939 static struct dmar_domain *
1940 find_domain(struct pci_dev *pdev)
1941 {
1942         struct device_domain_info *info;
1943
1944         /* No lock here, assumes no domain exit in normal case */
1945         info = pdev->dev.archdata.iommu;
1946         if (info)
1947                 return info->domain;
1948         return NULL;
1949 }
1950
1951 /* domain is initialized */
1952 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1953 {
1954         struct dmar_domain *domain, *found = NULL;
1955         struct intel_iommu *iommu;
1956         struct dmar_drhd_unit *drhd;
1957         struct device_domain_info *info, *tmp;
1958         struct pci_dev *dev_tmp;
1959         unsigned long flags;
1960         int bus = 0, devfn = 0;
1961         int segment;
1962         int ret;
1963
1964         domain = find_domain(pdev);
1965         if (domain)
1966                 return domain;
1967
1968         segment = pci_domain_nr(pdev->bus);
1969
1970         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1971         if (dev_tmp) {
1972                 if (pci_is_pcie(dev_tmp)) {
1973                         bus = dev_tmp->subordinate->number;
1974                         devfn = 0;
1975                 } else {
1976                         bus = dev_tmp->bus->number;
1977                         devfn = dev_tmp->devfn;
1978                 }
1979                 spin_lock_irqsave(&device_domain_lock, flags);
1980                 list_for_each_entry(info, &device_domain_list, global) {
1981                         if (info->segment == segment &&
1982                             info->bus == bus && info->devfn == devfn) {
1983                                 found = info->domain;
1984                                 break;
1985                         }
1986                 }
1987                 spin_unlock_irqrestore(&device_domain_lock, flags);
1988                 /* pcie-pci bridge already has a domain, uses it */
1989                 if (found) {
1990                         domain = found;
1991                         goto found_domain;
1992                 }
1993         }
1994
1995         domain = alloc_domain();
1996         if (!domain)
1997                 goto error;
1998
1999         /* Allocate new domain for the device */
2000         drhd = dmar_find_matched_drhd_unit(pdev);
2001         if (!drhd) {
2002                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2003                         pci_name(pdev));
2004                 return NULL;
2005         }
2006         iommu = drhd->iommu;
2007
2008         ret = iommu_attach_domain(domain, iommu);
2009         if (ret) {
2010                 free_domain_mem(domain);
2011                 goto error;
2012         }
2013
2014         if (domain_init(domain, gaw)) {
2015                 domain_exit(domain);
2016                 goto error;
2017         }
2018
2019         /* register pcie-to-pci device */
2020         if (dev_tmp) {
2021                 info = alloc_devinfo_mem();
2022                 if (!info) {
2023                         domain_exit(domain);
2024                         goto error;
2025                 }
2026                 info->segment = segment;
2027                 info->bus = bus;
2028                 info->devfn = devfn;
2029                 info->dev = NULL;
2030                 info->domain = domain;
2031                 /* This domain is shared by devices under p2p bridge */
2032                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2033
2034                 /* pcie-to-pci bridge already has a domain, uses it */
2035                 found = NULL;
2036                 spin_lock_irqsave(&device_domain_lock, flags);
2037                 list_for_each_entry(tmp, &device_domain_list, global) {
2038                         if (tmp->segment == segment &&
2039                             tmp->bus == bus && tmp->devfn == devfn) {
2040                                 found = tmp->domain;
2041                                 break;
2042                         }
2043                 }
2044                 if (found) {
2045                         spin_unlock_irqrestore(&device_domain_lock, flags);
2046                         free_devinfo_mem(info);
2047                         domain_exit(domain);
2048                         domain = found;
2049                 } else {
2050                         list_add(&info->link, &domain->devices);
2051                         list_add(&info->global, &device_domain_list);
2052                         spin_unlock_irqrestore(&device_domain_lock, flags);
2053                 }
2054         }
2055
2056 found_domain:
2057         info = alloc_devinfo_mem();
2058         if (!info)
2059                 goto error;
2060         info->segment = segment;
2061         info->bus = pdev->bus->number;
2062         info->devfn = pdev->devfn;
2063         info->dev = pdev;
2064         info->domain = domain;
2065         spin_lock_irqsave(&device_domain_lock, flags);
2066         /* somebody is fast */
2067         found = find_domain(pdev);
2068         if (found != NULL) {
2069                 spin_unlock_irqrestore(&device_domain_lock, flags);
2070                 if (found != domain) {
2071                         domain_exit(domain);
2072                         domain = found;
2073                 }
2074                 free_devinfo_mem(info);
2075                 return domain;
2076         }
2077         list_add(&info->link, &domain->devices);
2078         list_add(&info->global, &device_domain_list);
2079         pdev->dev.archdata.iommu = info;
2080         spin_unlock_irqrestore(&device_domain_lock, flags);
2081         return domain;
2082 error:
2083         /* recheck it here, maybe others set it */
2084         return find_domain(pdev);
2085 }
2086
2087 static int iommu_identity_mapping;
2088 #define IDENTMAP_ALL            1
2089 #define IDENTMAP_GFX            2
2090 #define IDENTMAP_AZALIA         4
2091
2092 static int iommu_domain_identity_map(struct dmar_domain *domain,
2093                                      unsigned long long start,
2094                                      unsigned long long end)
2095 {
2096         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2097         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2098
2099         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2100                           dma_to_mm_pfn(last_vpfn))) {
2101                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2102                 return -ENOMEM;
2103         }
2104
2105         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2106                  start, end, domain->id);
2107         /*
2108          * RMRR range might have overlap with physical memory range,
2109          * clear it first
2110          */
2111         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2112
2113         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2114                                   last_vpfn - first_vpfn + 1,
2115                                   DMA_PTE_READ|DMA_PTE_WRITE);
2116 }
2117
2118 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2119                                       unsigned long long start,
2120                                       unsigned long long end)
2121 {
2122         struct dmar_domain *domain;
2123         int ret;
2124
2125         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2126         if (!domain)
2127                 return -ENOMEM;
2128
2129         /* For _hardware_ passthrough, don't bother. But for software
2130            passthrough, we do it anyway -- it may indicate a memory
2131            range which is reserved in E820, so which didn't get set
2132            up to start with in si_domain */
2133         if (domain == si_domain && hw_pass_through) {
2134                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2135                        pci_name(pdev), start, end);
2136                 return 0;
2137         }
2138
2139         printk(KERN_INFO
2140                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2141                pci_name(pdev), start, end);
2142
2143         if (end < start) {
2144                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2145                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2146                         dmi_get_system_info(DMI_BIOS_VENDOR),
2147                         dmi_get_system_info(DMI_BIOS_VERSION),
2148                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2149                 ret = -EIO;
2150                 goto error;
2151         }
2152
2153         if (end >> agaw_to_width(domain->agaw)) {
2154                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2155                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2156                      agaw_to_width(domain->agaw),
2157                      dmi_get_system_info(DMI_BIOS_VENDOR),
2158                      dmi_get_system_info(DMI_BIOS_VERSION),
2159                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2160                 ret = -EIO;
2161                 goto error;
2162         }
2163
2164         ret = iommu_domain_identity_map(domain, start, end);
2165         if (ret)
2166                 goto error;
2167
2168         /* context entry init */
2169         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2170         if (ret)
2171                 goto error;
2172
2173         return 0;
2174
2175  error:
2176         domain_exit(domain);
2177         return ret;
2178 }
2179
2180 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2181         struct pci_dev *pdev)
2182 {
2183         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2184                 return 0;
2185         return iommu_prepare_identity_map(pdev, rmrr->base_address,
2186                 rmrr->end_address);
2187 }
2188
2189 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2190 static inline void iommu_prepare_isa(void)
2191 {
2192         struct pci_dev *pdev;
2193         int ret;
2194
2195         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2196         if (!pdev)
2197                 return;
2198
2199         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2200         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2201
2202         if (ret)
2203                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2204                        "floppy might not work\n");
2205
2206 }
2207 #else
2208 static inline void iommu_prepare_isa(void)
2209 {
2210         return;
2211 }
2212 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2213
2214 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2215
2216 static int __init si_domain_init(int hw)
2217 {
2218         struct dmar_drhd_unit *drhd;
2219         struct intel_iommu *iommu;
2220         int nid, ret = 0;
2221
2222         si_domain = alloc_domain();
2223         if (!si_domain)
2224                 return -EFAULT;
2225
2226         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2227
2228         for_each_active_iommu(iommu, drhd) {
2229                 ret = iommu_attach_domain(si_domain, iommu);
2230                 if (ret) {
2231                         domain_exit(si_domain);
2232                         return -EFAULT;
2233                 }
2234         }
2235
2236         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2237                 domain_exit(si_domain);
2238                 return -EFAULT;
2239         }
2240
2241         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2242
2243         if (hw)
2244                 return 0;
2245
2246         for_each_online_node(nid) {
2247                 unsigned long start_pfn, end_pfn;
2248                 int i;
2249
2250                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2251                         ret = iommu_domain_identity_map(si_domain,
2252                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2253                         if (ret)
2254                                 return ret;
2255                 }
2256         }
2257
2258         return 0;
2259 }
2260
2261 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2262                                           struct pci_dev *pdev);
2263 static int identity_mapping(struct pci_dev *pdev)
2264 {
2265         struct device_domain_info *info;
2266
2267         if (likely(!iommu_identity_mapping))
2268                 return 0;
2269
2270         info = pdev->dev.archdata.iommu;
2271         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2272                 return (info->domain == si_domain);
2273
2274         return 0;
2275 }
2276
2277 static int domain_add_dev_info(struct dmar_domain *domain,
2278                                struct pci_dev *pdev,
2279                                int translation)
2280 {
2281         struct device_domain_info *info;
2282         unsigned long flags;
2283         int ret;
2284
2285         info = alloc_devinfo_mem();
2286         if (!info)
2287                 return -ENOMEM;
2288
2289         ret = domain_context_mapping(domain, pdev, translation);
2290         if (ret) {
2291                 free_devinfo_mem(info);
2292                 return ret;
2293         }
2294
2295         info->segment = pci_domain_nr(pdev->bus);
2296         info->bus = pdev->bus->number;
2297         info->devfn = pdev->devfn;
2298         info->dev = pdev;
2299         info->domain = domain;
2300
2301         spin_lock_irqsave(&device_domain_lock, flags);
2302         list_add(&info->link, &domain->devices);
2303         list_add(&info->global, &device_domain_list);
2304         pdev->dev.archdata.iommu = info;
2305         spin_unlock_irqrestore(&device_domain_lock, flags);
2306
2307         return 0;
2308 }
2309
2310 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2311 {
2312         if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2313                 return 1;
2314
2315         if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2316                 return 1;
2317
2318         if (!(iommu_identity_mapping & IDENTMAP_ALL))
2319                 return 0;
2320
2321         /*
2322          * We want to start off with all devices in the 1:1 domain, and
2323          * take them out later if we find they can't access all of memory.
2324          *
2325          * However, we can't do this for PCI devices behind bridges,
2326          * because all PCI devices behind the same bridge will end up
2327          * with the same source-id on their transactions.
2328          *
2329          * Practically speaking, we can't change things around for these
2330          * devices at run-time, because we can't be sure there'll be no
2331          * DMA transactions in flight for any of their siblings.
2332          *
2333          * So PCI devices (unless they're on the root bus) as well as
2334          * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2335          * the 1:1 domain, just in _case_ one of their siblings turns out
2336          * not to be able to map all of memory.
2337          */
2338         if (!pci_is_pcie(pdev)) {
2339                 if (!pci_is_root_bus(pdev->bus))
2340                         return 0;
2341                 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2342                         return 0;
2343         } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2344                 return 0;
2345
2346         /*
2347          * At boot time, we don't yet know if devices will be 64-bit capable.
2348          * Assume that they will -- if they turn out not to be, then we can
2349          * take them out of the 1:1 domain later.
2350          */
2351         if (!startup) {
2352                 /*
2353                  * If the device's dma_mask is less than the system's memory
2354                  * size then this is not a candidate for identity mapping.
2355                  */
2356                 u64 dma_mask = pdev->dma_mask;
2357
2358                 if (pdev->dev.coherent_dma_mask &&
2359                     pdev->dev.coherent_dma_mask < dma_mask)
2360                         dma_mask = pdev->dev.coherent_dma_mask;
2361
2362                 return dma_mask >= dma_get_required_mask(&pdev->dev);
2363         }
2364
2365         return 1;
2366 }
2367
2368 static int __init iommu_prepare_static_identity_mapping(int hw)
2369 {
2370         struct pci_dev *pdev = NULL;
2371         int ret;
2372
2373         ret = si_domain_init(hw);
2374         if (ret)
2375                 return -EFAULT;
2376
2377         for_each_pci_dev(pdev) {
2378                 if (iommu_should_identity_map(pdev, 1)) {
2379                         ret = domain_add_dev_info(si_domain, pdev,
2380                                              hw ? CONTEXT_TT_PASS_THROUGH :
2381                                                   CONTEXT_TT_MULTI_LEVEL);
2382                         if (ret) {
2383                                 /* device not associated with an iommu */
2384                                 if (ret == -ENODEV)
2385                                         continue;
2386                                 return ret;
2387                         }
2388                         pr_info("IOMMU: %s identity mapping for device %s\n",
2389                                 hw ? "hardware" : "software", pci_name(pdev));
2390                 }
2391         }
2392
2393         return 0;
2394 }
2395
2396 static int __init init_dmars(void)
2397 {
2398         struct dmar_drhd_unit *drhd;
2399         struct dmar_rmrr_unit *rmrr;
2400         struct pci_dev *pdev;
2401         struct intel_iommu *iommu;
2402         int i, ret;
2403
2404         /*
2405          * for each drhd
2406          *    allocate root
2407          *    initialize and program root entry to not present
2408          * endfor
2409          */
2410         for_each_drhd_unit(drhd) {
2411                 /*
2412                  * lock not needed as this is only incremented in the single
2413                  * threaded kernel __init code path all other access are read
2414                  * only
2415                  */
2416                 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2417                         g_num_of_iommus++;
2418                         continue;
2419                 }
2420                 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2421                           IOMMU_UNITS_SUPPORTED);
2422         }
2423
2424         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2425                         GFP_KERNEL);
2426         if (!g_iommus) {
2427                 printk(KERN_ERR "Allocating global iommu array failed\n");
2428                 ret = -ENOMEM;
2429                 goto error;
2430         }
2431
2432         deferred_flush = kzalloc(g_num_of_iommus *
2433                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2434         if (!deferred_flush) {
2435                 ret = -ENOMEM;
2436                 goto error;
2437         }
2438
2439         for_each_drhd_unit(drhd) {
2440                 if (drhd->ignored)
2441                         continue;
2442
2443                 iommu = drhd->iommu;
2444                 g_iommus[iommu->seq_id] = iommu;
2445
2446                 ret = iommu_init_domains(iommu);
2447                 if (ret)
2448                         goto error;
2449
2450                 /*
2451                  * TBD:
2452                  * we could share the same root & context tables
2453                  * among all IOMMU's. Need to Split it later.
2454                  */
2455                 ret = iommu_alloc_root_entry(iommu);
2456                 if (ret) {
2457                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2458                         goto error;
2459                 }
2460                 if (!ecap_pass_through(iommu->ecap))
2461                         hw_pass_through = 0;
2462         }
2463
2464         /*
2465          * Start from the sane iommu hardware state.
2466          */
2467         for_each_drhd_unit(drhd) {
2468                 if (drhd->ignored)
2469                         continue;
2470
2471                 iommu = drhd->iommu;
2472
2473                 /*
2474                  * If the queued invalidation is already initialized by us
2475                  * (for example, while enabling interrupt-remapping) then
2476                  * we got the things already rolling from a sane state.
2477                  */
2478                 if (iommu->qi)
2479                         continue;
2480
2481                 /*
2482                  * Clear any previous faults.
2483                  */
2484                 dmar_fault(-1, iommu);
2485                 /*
2486                  * Disable queued invalidation if supported and already enabled
2487                  * before OS handover.
2488                  */
2489                 dmar_disable_qi(iommu);
2490         }
2491
2492         for_each_drhd_unit(drhd) {
2493                 if (drhd->ignored)
2494                         continue;
2495
2496                 iommu = drhd->iommu;
2497
2498                 if (dmar_enable_qi(iommu)) {
2499                         /*
2500                          * Queued Invalidate not enabled, use Register Based
2501                          * Invalidate
2502                          */
2503                         iommu->flush.flush_context = __iommu_flush_context;
2504                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2505                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2506                                "invalidation\n",
2507                                 iommu->seq_id,
2508                                (unsigned long long)drhd->reg_base_addr);
2509                 } else {
2510                         iommu->flush.flush_context = qi_flush_context;
2511                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2512                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2513                                "invalidation\n",
2514                                 iommu->seq_id,
2515                                (unsigned long long)drhd->reg_base_addr);
2516                 }
2517         }
2518
2519         if (iommu_pass_through)
2520                 iommu_identity_mapping |= IDENTMAP_ALL;
2521
2522 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2523         iommu_identity_mapping |= IDENTMAP_GFX;
2524 #endif
2525
2526         check_tylersburg_isoch();
2527
2528         /*
2529          * If pass through is not set or not enabled, setup context entries for
2530          * identity mappings for rmrr, gfx, and isa and may fall back to static
2531          * identity mapping if iommu_identity_mapping is set.
2532          */
2533         if (iommu_identity_mapping) {
2534                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2535                 if (ret) {
2536                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2537                         goto error;
2538                 }
2539         }
2540         /*
2541          * For each rmrr
2542          *   for each dev attached to rmrr
2543          *   do
2544          *     locate drhd for dev, alloc domain for dev
2545          *     allocate free domain
2546          *     allocate page table entries for rmrr
2547          *     if context not allocated for bus
2548          *           allocate and init context
2549          *           set present in root table for this bus
2550          *     init context with domain, translation etc
2551          *    endfor
2552          * endfor
2553          */
2554         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2555         for_each_rmrr_units(rmrr) {
2556                 for (i = 0; i < rmrr->devices_cnt; i++) {
2557                         pdev = rmrr->devices[i];
2558                         /*
2559                          * some BIOS lists non-exist devices in DMAR
2560                          * table.
2561                          */
2562                         if (!pdev)
2563                                 continue;
2564                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2565                         if (ret)
2566                                 printk(KERN_ERR
2567                                        "IOMMU: mapping reserved region failed\n");
2568                 }
2569         }
2570
2571         iommu_prepare_isa();
2572
2573         /*
2574          * for each drhd
2575          *   enable fault log
2576          *   global invalidate context cache
2577          *   global invalidate iotlb
2578          *   enable translation
2579          */
2580         for_each_drhd_unit(drhd) {
2581                 if (drhd->ignored) {
2582                         /*
2583                          * we always have to disable PMRs or DMA may fail on
2584                          * this device
2585                          */
2586                         if (force_on)
2587                                 iommu_disable_protect_mem_regions(drhd->iommu);
2588                         continue;
2589                 }
2590                 iommu = drhd->iommu;
2591
2592                 iommu_flush_write_buffer(iommu);
2593
2594                 ret = dmar_set_interrupt(iommu);
2595                 if (ret)
2596                         goto error;
2597
2598                 iommu_set_root_entry(iommu);
2599
2600                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2601                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2602
2603                 ret = iommu_enable_translation(iommu);
2604                 if (ret)
2605                         goto error;
2606
2607                 iommu_disable_protect_mem_regions(iommu);
2608         }
2609
2610         return 0;
2611 error:
2612         for_each_drhd_unit(drhd) {
2613                 if (drhd->ignored)
2614                         continue;
2615                 iommu = drhd->iommu;
2616                 free_iommu(iommu);
2617         }
2618         kfree(g_iommus);
2619         return ret;
2620 }
2621
2622 /* This takes a number of _MM_ pages, not VTD pages */
2623 static struct iova *intel_alloc_iova(struct device *dev,
2624                                      struct dmar_domain *domain,
2625                                      unsigned long nrpages, uint64_t dma_mask)
2626 {
2627         struct pci_dev *pdev = to_pci_dev(dev);
2628         struct iova *iova = NULL;
2629
2630         /* Restrict dma_mask to the width that the iommu can handle */
2631         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2632
2633         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2634                 /*
2635                  * First try to allocate an io virtual address in
2636                  * DMA_BIT_MASK(32) and if that fails then try allocating
2637                  * from higher range
2638                  */
2639                 iova = alloc_iova(&domain->iovad, nrpages,
2640                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2641                 if (iova)
2642                         return iova;
2643         }
2644         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2645         if (unlikely(!iova)) {
2646                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2647                        nrpages, pci_name(pdev));
2648                 return NULL;
2649         }
2650
2651         return iova;
2652 }
2653
2654 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2655 {
2656         struct dmar_domain *domain;
2657         int ret;
2658
2659         domain = get_domain_for_dev(pdev,
2660                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2661         if (!domain) {
2662                 printk(KERN_ERR
2663                         "Allocating domain for %s failed", pci_name(pdev));
2664                 return NULL;
2665         }
2666
2667         /* make sure context mapping is ok */
2668         if (unlikely(!domain_context_mapped(pdev))) {
2669                 ret = domain_context_mapping(domain, pdev,
2670                                              CONTEXT_TT_MULTI_LEVEL);
2671                 if (ret) {
2672                         printk(KERN_ERR
2673                                 "Domain context map for %s failed",
2674                                 pci_name(pdev));
2675                         return NULL;
2676                 }
2677         }
2678
2679         return domain;
2680 }
2681
2682 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2683 {
2684         struct device_domain_info *info;
2685
2686         /* No lock here, assumes no domain exit in normal case */
2687         info = dev->dev.archdata.iommu;
2688         if (likely(info))
2689                 return info->domain;
2690
2691         return __get_valid_domain_for_dev(dev);
2692 }
2693
2694 static int iommu_dummy(struct pci_dev *pdev)
2695 {
2696         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2697 }
2698
2699 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2700 static int iommu_no_mapping(struct device *dev)
2701 {
2702         struct pci_dev *pdev;
2703         int found;
2704
2705         if (unlikely(dev->bus != &pci_bus_type))
2706                 return 1;
2707
2708         pdev = to_pci_dev(dev);
2709         if (iommu_dummy(pdev))
2710                 return 1;
2711
2712         if (!iommu_identity_mapping)
2713                 return 0;
2714
2715         found = identity_mapping(pdev);
2716         if (found) {
2717                 if (iommu_should_identity_map(pdev, 0))
2718                         return 1;
2719                 else {
2720                         /*
2721                          * 32 bit DMA is removed from si_domain and fall back
2722                          * to non-identity mapping.
2723                          */
2724                         domain_remove_one_dev_info(si_domain, pdev);
2725                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2726                                pci_name(pdev));
2727                         return 0;
2728                 }
2729         } else {
2730                 /*
2731                  * In case of a detached 64 bit DMA device from vm, the device
2732                  * is put into si_domain for identity mapping.
2733                  */
2734                 if (iommu_should_identity_map(pdev, 0)) {
2735                         int ret;
2736                         ret = domain_add_dev_info(si_domain, pdev,
2737                                                   hw_pass_through ?
2738                                                   CONTEXT_TT_PASS_THROUGH :
2739                                                   CONTEXT_TT_MULTI_LEVEL);
2740                         if (!ret) {
2741                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2742                                        pci_name(pdev));
2743                                 return 1;
2744                         }
2745                 }
2746         }
2747
2748         return 0;
2749 }
2750
2751 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2752                                      size_t size, int dir, u64 dma_mask)
2753 {
2754         struct pci_dev *pdev = to_pci_dev(hwdev);
2755         struct dmar_domain *domain;
2756         phys_addr_t start_paddr;
2757         struct iova *iova;
2758         int prot = 0;
2759         int ret;
2760         struct intel_iommu *iommu;
2761         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2762
2763         BUG_ON(dir == DMA_NONE);
2764
2765         if (iommu_no_mapping(hwdev))
2766                 return paddr;
2767
2768         domain = get_valid_domain_for_dev(pdev);
2769         if (!domain)
2770                 return 0;
2771
2772         iommu = domain_get_iommu(domain);
2773         size = aligned_nrpages(paddr, size);
2774
2775         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2776         if (!iova)
2777                 goto error;
2778
2779         /*
2780          * Check if DMAR supports zero-length reads on write only
2781          * mappings..
2782          */
2783         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2784                         !cap_zlr(iommu->cap))
2785                 prot |= DMA_PTE_READ;
2786         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2787                 prot |= DMA_PTE_WRITE;
2788         /*
2789          * paddr - (paddr + size) might be partial page, we should map the whole
2790          * page.  Note: if two part of one page are separately mapped, we
2791          * might have two guest_addr mapping to the same host paddr, but this
2792          * is not a big problem
2793          */
2794         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2795                                  mm_to_dma_pfn(paddr_pfn), size, prot);
2796         if (ret)
2797                 goto error;
2798
2799         /* it's a non-present to present mapping. Only flush if caching mode */
2800         if (cap_caching_mode(iommu->cap))
2801                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2802         else
2803                 iommu_flush_write_buffer(iommu);
2804
2805         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2806         start_paddr += paddr & ~PAGE_MASK;
2807         return start_paddr;
2808
2809 error:
2810         if (iova)
2811                 __free_iova(&domain->iovad, iova);
2812         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2813                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2814         return 0;
2815 }
2816
2817 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2818                                  unsigned long offset, size_t size,
2819                                  enum dma_data_direction dir,
2820                                  struct dma_attrs *attrs)
2821 {
2822         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2823                                   dir, to_pci_dev(dev)->dma_mask);
2824 }
2825
2826 static void flush_unmaps(void)
2827 {
2828         int i, j;
2829
2830         timer_on = 0;
2831
2832         /* just flush them all */
2833         for (i = 0; i < g_num_of_iommus; i++) {
2834                 struct intel_iommu *iommu = g_iommus[i];
2835                 if (!iommu)
2836                         continue;
2837
2838                 if (!deferred_flush[i].next)
2839                         continue;
2840
2841                 /* In caching mode, global flushes turn emulation expensive */
2842                 if (!cap_caching_mode(iommu->cap))
2843                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2844                                          DMA_TLB_GLOBAL_FLUSH);
2845                 for (j = 0; j < deferred_flush[i].next; j++) {
2846                         unsigned long mask;
2847                         struct iova *iova = deferred_flush[i].iova[j];
2848                         struct dmar_domain *domain = deferred_flush[i].domain[j];
2849
2850                         /* On real hardware multiple invalidations are expensive */
2851                         if (cap_caching_mode(iommu->cap))
2852                                 iommu_flush_iotlb_psi(iommu, domain->id,
2853                                 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2854                         else {
2855                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2856                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2857                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2858                         }
2859                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2860                 }
2861                 deferred_flush[i].next = 0;
2862         }
2863
2864         list_size = 0;
2865 }
2866
2867 static void flush_unmaps_timeout(unsigned long data)
2868 {
2869         unsigned long flags;
2870
2871         spin_lock_irqsave(&async_umap_flush_lock, flags);
2872         flush_unmaps();
2873         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2874 }
2875
2876 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2877 {
2878         unsigned long flags;
2879         int next, iommu_id;
2880         struct intel_iommu *iommu;
2881
2882         spin_lock_irqsave(&async_umap_flush_lock, flags);
2883         if (list_size == HIGH_WATER_MARK)
2884                 flush_unmaps();
2885
2886         iommu = domain_get_iommu(dom);
2887         iommu_id = iommu->seq_id;
2888
2889         next = deferred_flush[iommu_id].next;
2890         deferred_flush[iommu_id].domain[next] = dom;
2891         deferred_flush[iommu_id].iova[next] = iova;
2892         deferred_flush[iommu_id].next++;
2893
2894         if (!timer_on) {
2895                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2896                 timer_on = 1;
2897         }
2898         list_size++;
2899         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2900 }
2901
2902 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2903                              size_t size, enum dma_data_direction dir,
2904                              struct dma_attrs *attrs)
2905 {
2906         struct pci_dev *pdev = to_pci_dev(dev);
2907         struct dmar_domain *domain;
2908         unsigned long start_pfn, last_pfn;
2909         struct iova *iova;
2910         struct intel_iommu *iommu;
2911
2912         if (iommu_no_mapping(dev))
2913                 return;
2914
2915         domain = find_domain(pdev);
2916         BUG_ON(!domain);
2917
2918         iommu = domain_get_iommu(domain);
2919
2920         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2921         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2922                       (unsigned long long)dev_addr))
2923                 return;
2924
2925         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2926         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2927
2928         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2929                  pci_name(pdev), start_pfn, last_pfn);
2930
2931         /*  clear the whole page */
2932         dma_pte_clear_range(domain, start_pfn, last_pfn);
2933
2934         /* free page tables */
2935         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2936
2937         if (intel_iommu_strict) {
2938                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2939                                       last_pfn - start_pfn + 1, 0);
2940                 /* free iova */
2941                 __free_iova(&domain->iovad, iova);
2942         } else {
2943                 add_unmap(domain, iova);
2944                 /*
2945                  * queue up the release of the unmap to save the 1/6th of the
2946                  * cpu used up by the iotlb flush operation...
2947                  */
2948         }
2949 }
2950
2951 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2952                                   dma_addr_t *dma_handle, gfp_t flags,
2953                                   struct dma_attrs *attrs)
2954 {
2955         void *vaddr;
2956         int order;
2957
2958         size = PAGE_ALIGN(size);
2959         order = get_order(size);
2960
2961         if (!iommu_no_mapping(hwdev))
2962                 flags &= ~(GFP_DMA | GFP_DMA32);
2963         else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2964                 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2965                         flags |= GFP_DMA;
2966                 else
2967                         flags |= GFP_DMA32;
2968         }
2969
2970         vaddr = (void *)__get_free_pages(flags, order);
2971         if (!vaddr)
2972                 return NULL;
2973         memset(vaddr, 0, size);
2974
2975         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2976                                          DMA_BIDIRECTIONAL,
2977                                          hwdev->coherent_dma_mask);
2978         if (*dma_handle)
2979                 return vaddr;
2980         free_pages((unsigned long)vaddr, order);
2981         return NULL;
2982 }
2983
2984 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2985                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
2986 {
2987         int order;
2988
2989         size = PAGE_ALIGN(size);
2990         order = get_order(size);
2991
2992         intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2993         free_pages((unsigned long)vaddr, order);
2994 }
2995
2996 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2997                            int nelems, enum dma_data_direction dir,
2998                            struct dma_attrs *attrs)
2999 {
3000         struct pci_dev *pdev = to_pci_dev(hwdev);
3001         struct dmar_domain *domain;
3002         unsigned long start_pfn, last_pfn;
3003         struct iova *iova;
3004         struct intel_iommu *iommu;
3005
3006         if (iommu_no_mapping(hwdev))
3007                 return;
3008
3009         domain = find_domain(pdev);
3010         BUG_ON(!domain);
3011
3012         iommu = domain_get_iommu(domain);
3013
3014         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3015         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3016                       (unsigned long long)sglist[0].dma_address))
3017                 return;
3018
3019         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3020         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3021
3022         /*  clear the whole page */
3023         dma_pte_clear_range(domain, start_pfn, last_pfn);
3024
3025         /* free page tables */
3026         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
3027
3028         if (intel_iommu_strict) {
3029                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3030                                       last_pfn - start_pfn + 1, 0);
3031                 /* free iova */
3032                 __free_iova(&domain->iovad, iova);
3033         } else {
3034                 add_unmap(domain, iova);
3035                 /*
3036                  * queue up the release of the unmap to save the 1/6th of the
3037                  * cpu used up by the iotlb flush operation...
3038                  */
3039         }
3040 }
3041
3042 static int intel_nontranslate_map_sg(struct device *hddev,
3043         struct scatterlist *sglist, int nelems, int dir)
3044 {
3045         int i;
3046         struct scatterlist *sg;
3047
3048         for_each_sg(sglist, sg, nelems, i) {
3049                 BUG_ON(!sg_page(sg));
3050                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3051                 sg->dma_length = sg->length;
3052         }
3053         return nelems;
3054 }
3055
3056 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3057                         enum dma_data_direction dir, struct dma_attrs *attrs)
3058 {
3059         int i;
3060         struct pci_dev *pdev = to_pci_dev(hwdev);
3061         struct dmar_domain *domain;
3062         size_t size = 0;
3063         int prot = 0;
3064         struct iova *iova = NULL;
3065         int ret;
3066         struct scatterlist *sg;
3067         unsigned long start_vpfn;
3068         struct intel_iommu *iommu;
3069
3070         BUG_ON(dir == DMA_NONE);
3071         if (iommu_no_mapping(hwdev))
3072                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3073
3074         domain = get_valid_domain_for_dev(pdev);
3075         if (!domain)
3076                 return 0;
3077
3078         iommu = domain_get_iommu(domain);
3079
3080         for_each_sg(sglist, sg, nelems, i)
3081                 size += aligned_nrpages(sg->offset, sg->length);
3082
3083         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3084                                 pdev->dma_mask);
3085         if (!iova) {
3086                 sglist->dma_length = 0;
3087                 return 0;
3088         }
3089
3090         /*
3091          * Check if DMAR supports zero-length reads on write only
3092          * mappings..
3093          */
3094         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3095                         !cap_zlr(iommu->cap))
3096                 prot |= DMA_PTE_READ;
3097         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3098                 prot |= DMA_PTE_WRITE;
3099
3100         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3101
3102         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3103         if (unlikely(ret)) {
3104                 /*  clear the page */
3105                 dma_pte_clear_range(domain, start_vpfn,
3106                                     start_vpfn + size - 1);
3107                 /* free page tables */
3108                 dma_pte_free_pagetable(domain, start_vpfn,
3109                                        start_vpfn + size - 1);
3110                 /* free iova */
3111                 __free_iova(&domain->iovad, iova);
3112                 return 0;
3113         }
3114
3115         /* it's a non-present to present mapping. Only flush if caching mode */
3116         if (cap_caching_mode(iommu->cap))
3117                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3118         else
3119                 iommu_flush_write_buffer(iommu);
3120
3121         return nelems;
3122 }
3123
3124 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3125 {
3126         return !dma_addr;
3127 }
3128
3129 struct dma_map_ops intel_dma_ops = {
3130         .alloc = intel_alloc_coherent,
3131         .free = intel_free_coherent,
3132         .map_sg = intel_map_sg,
3133         .unmap_sg = intel_unmap_sg,
3134         .map_page = intel_map_page,
3135         .unmap_page = intel_unmap_page,
3136         .mapping_error = intel_mapping_error,
3137 };
3138
3139 static inline int iommu_domain_cache_init(void)
3140 {
3141         int ret = 0;
3142
3143         iommu_domain_cache = kmem_cache_create("iommu_domain",
3144                                          sizeof(struct dmar_domain),
3145                                          0,
3146                                          SLAB_HWCACHE_ALIGN,
3147
3148                                          NULL);
3149         if (!iommu_domain_cache) {
3150                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3151                 ret = -ENOMEM;
3152         }
3153
3154         return ret;
3155 }
3156
3157 static inline int iommu_devinfo_cache_init(void)
3158 {
3159         int ret = 0;
3160
3161         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3162                                          sizeof(struct device_domain_info),
3163                                          0,
3164                                          SLAB_HWCACHE_ALIGN,
3165                                          NULL);
3166         if (!iommu_devinfo_cache) {
3167                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3168                 ret = -ENOMEM;
3169         }
3170
3171         return ret;
3172 }
3173
3174 static inline int iommu_iova_cache_init(void)
3175 {
3176         int ret = 0;
3177
3178         iommu_iova_cache = kmem_cache_create("iommu_iova",
3179                                          sizeof(struct iova),
3180                                          0,
3181                                          SLAB_HWCACHE_ALIGN,
3182                                          NULL);
3183         if (!iommu_iova_cache) {
3184                 printk(KERN_ERR "Couldn't create iova cache\n");
3185                 ret = -ENOMEM;
3186         }
3187
3188         return ret;
3189 }
3190
3191 static int __init iommu_init_mempool(void)
3192 {
3193         int ret;
3194         ret = iommu_iova_cache_init();
3195         if (ret)
3196                 return ret;
3197
3198         ret = iommu_domain_cache_init();
3199         if (ret)
3200                 goto domain_error;
3201
3202         ret = iommu_devinfo_cache_init();
3203         if (!ret)
3204                 return ret;
3205
3206         kmem_cache_destroy(iommu_domain_cache);
3207 domain_error:
3208         kmem_cache_destroy(iommu_iova_cache);
3209
3210         return -ENOMEM;
3211 }
3212
3213 static void __init iommu_exit_mempool(void)
3214 {
3215         kmem_cache_destroy(iommu_devinfo_cache);
3216         kmem_cache_destroy(iommu_domain_cache);
3217         kmem_cache_destroy(iommu_iova_cache);
3218
3219 }
3220
3221 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3222 {
3223         struct dmar_drhd_unit *drhd;
3224         u32 vtbar;
3225         int rc;
3226
3227         /* We know that this device on this chipset has its own IOMMU.
3228          * If we find it under a different IOMMU, then the BIOS is lying
3229          * to us. Hope that the IOMMU for this device is actually
3230          * disabled, and it needs no translation...
3231          */
3232         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3233         if (rc) {
3234                 /* "can't" happen */
3235                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3236                 return;
3237         }
3238         vtbar &= 0xffff0000;
3239
3240         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3241         drhd = dmar_find_matched_drhd_unit(pdev);
3242         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3243                             TAINT_FIRMWARE_WORKAROUND,
3244                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3245                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3246 }
3247 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3248
3249 static void __init init_no_remapping_devices(void)
3250 {
3251         struct dmar_drhd_unit *drhd;
3252
3253         for_each_drhd_unit(drhd) {
3254                 if (!drhd->include_all) {
3255                         int i;
3256                         for (i = 0; i < drhd->devices_cnt; i++)
3257                                 if (drhd->devices[i] != NULL)
3258                                         break;
3259                         /* ignore DMAR unit if no pci devices exist */
3260                         if (i == drhd->devices_cnt)
3261                                 drhd->ignored = 1;
3262                 }
3263         }
3264
3265         for_each_drhd_unit(drhd) {
3266                 int i;
3267                 if (drhd->ignored || drhd->include_all)
3268                         continue;
3269
3270                 for (i = 0; i < drhd->devices_cnt; i++)
3271                         if (drhd->devices[i] &&
3272                             !IS_GFX_DEVICE(drhd->devices[i]))
3273                                 break;
3274
3275                 if (i < drhd->devices_cnt)
3276                         continue;
3277
3278                 /* This IOMMU has *only* gfx devices. Either bypass it or
3279                    set the gfx_mapped flag, as appropriate */
3280                 if (dmar_map_gfx) {
3281                         intel_iommu_gfx_mapped = 1;
3282                 } else {
3283                         drhd->ignored = 1;
3284                         for (i = 0; i < drhd->devices_cnt; i++) {
3285                                 if (!drhd->devices[i])
3286                                         continue;
3287                                 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3288                         }
3289                 }
3290         }
3291 }
3292
3293 #ifdef CONFIG_SUSPEND
3294 static int init_iommu_hw(void)
3295 {
3296         struct dmar_drhd_unit *drhd;
3297         struct intel_iommu *iommu = NULL;
3298
3299         for_each_active_iommu(iommu, drhd)
3300                 if (iommu->qi)
3301                         dmar_reenable_qi(iommu);
3302
3303         for_each_iommu(iommu, drhd) {
3304                 if (drhd->ignored) {
3305                         /*
3306                          * we always have to disable PMRs or DMA may fail on
3307                          * this device
3308                          */
3309                         if (force_on)
3310                                 iommu_disable_protect_mem_regions(iommu);
3311                         continue;
3312                 }
3313
3314                 iommu_flush_write_buffer(iommu);
3315
3316                 iommu_set_root_entry(iommu);
3317
3318                 iommu->flush.flush_context(iommu, 0, 0, 0,
3319                                            DMA_CCMD_GLOBAL_INVL);
3320                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3321                                          DMA_TLB_GLOBAL_FLUSH);
3322                 if (iommu_enable_translation(iommu))
3323                         return 1;
3324                 iommu_disable_protect_mem_regions(iommu);
3325         }
3326
3327         return 0;
3328 }
3329
3330 static void iommu_flush_all(void)
3331 {
3332         struct dmar_drhd_unit *drhd;
3333         struct intel_iommu *iommu;
3334
3335         for_each_active_iommu(iommu, drhd) {
3336                 iommu->flush.flush_context(iommu, 0, 0, 0,
3337                                            DMA_CCMD_GLOBAL_INVL);
3338                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3339                                          DMA_TLB_GLOBAL_FLUSH);
3340         }
3341 }
3342
3343 static int iommu_suspend(void)
3344 {
3345         struct dmar_drhd_unit *drhd;
3346         struct intel_iommu *iommu = NULL;
3347         unsigned long flag;
3348
3349         for_each_active_iommu(iommu, drhd) {
3350                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3351                                                  GFP_ATOMIC);
3352                 if (!iommu->iommu_state)
3353                         goto nomem;
3354         }
3355
3356         iommu_flush_all();
3357
3358         for_each_active_iommu(iommu, drhd) {
3359                 iommu_disable_translation(iommu);
3360
3361                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3362
3363                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3364                         readl(iommu->reg + DMAR_FECTL_REG);
3365                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3366                         readl(iommu->reg + DMAR_FEDATA_REG);
3367                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3368                         readl(iommu->reg + DMAR_FEADDR_REG);
3369                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3370                         readl(iommu->reg + DMAR_FEUADDR_REG);
3371
3372                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3373         }
3374         return 0;
3375
3376 nomem:
3377         for_each_active_iommu(iommu, drhd)
3378                 kfree(iommu->iommu_state);
3379
3380         return -ENOMEM;
3381 }
3382
3383 static void iommu_resume(void)
3384 {
3385         struct dmar_drhd_unit *drhd;
3386         struct intel_iommu *iommu = NULL;
3387         unsigned long flag;
3388
3389         if (init_iommu_hw()) {
3390                 if (force_on)
3391                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3392                 else
3393                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3394                 return;
3395         }
3396
3397         for_each_active_iommu(iommu, drhd) {
3398
3399                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3400
3401                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3402                         iommu->reg + DMAR_FECTL_REG);
3403                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3404                         iommu->reg + DMAR_FEDATA_REG);
3405                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3406                         iommu->reg + DMAR_FEADDR_REG);
3407                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3408                         iommu->reg + DMAR_FEUADDR_REG);
3409
3410                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3411         }
3412
3413         for_each_active_iommu(iommu, drhd)
3414                 kfree(iommu->iommu_state);
3415 }
3416
3417 static struct syscore_ops iommu_syscore_ops = {
3418         .resume         = iommu_resume,
3419         .suspend        = iommu_suspend,
3420 };
3421
3422 static void __init init_iommu_pm_ops(void)
3423 {
3424         register_syscore_ops(&iommu_syscore_ops);
3425 }
3426
3427 #else
3428 static inline void init_iommu_pm_ops(void) {}
3429 #endif  /* CONFIG_PM */
3430
3431 LIST_HEAD(dmar_rmrr_units);
3432
3433 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3434 {
3435         list_add(&rmrr->list, &dmar_rmrr_units);
3436 }
3437
3438
3439 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3440 {
3441         struct acpi_dmar_reserved_memory *rmrr;
3442         struct dmar_rmrr_unit *rmrru;
3443
3444         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3445         if (!rmrru)
3446                 return -ENOMEM;
3447
3448         rmrru->hdr = header;
3449         rmrr = (struct acpi_dmar_reserved_memory *)header;
3450         rmrru->base_address = rmrr->base_address;
3451         rmrru->end_address = rmrr->end_address;
3452
3453         dmar_register_rmrr_unit(rmrru);
3454         return 0;
3455 }
3456
3457 static int __init
3458 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3459 {
3460         struct acpi_dmar_reserved_memory *rmrr;
3461         int ret;
3462
3463         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3464         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3465                 ((void *)rmrr) + rmrr->header.length,
3466                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3467
3468         if (ret || (rmrru->devices_cnt == 0)) {
3469                 list_del(&rmrru->list);
3470                 kfree(rmrru);
3471         }
3472         return ret;
3473 }
3474
3475 static LIST_HEAD(dmar_atsr_units);
3476
3477 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3478 {
3479         struct acpi_dmar_atsr *atsr;
3480         struct dmar_atsr_unit *atsru;
3481
3482         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3483         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3484         if (!atsru)
3485                 return -ENOMEM;
3486
3487         atsru->hdr = hdr;
3488         atsru->include_all = atsr->flags & 0x1;
3489
3490         list_add(&atsru->list, &dmar_atsr_units);
3491
3492         return 0;
3493 }
3494
3495 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3496 {
3497         int rc;
3498         struct acpi_dmar_atsr *atsr;
3499
3500         if (atsru->include_all)
3501                 return 0;
3502
3503         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3504         rc = dmar_parse_dev_scope((void *)(atsr + 1),
3505                                 (void *)atsr + atsr->header.length,
3506                                 &atsru->devices_cnt, &atsru->devices,
3507                                 atsr->segment);
3508         if (rc || !atsru->devices_cnt) {
3509                 list_del(&atsru->list);
3510                 kfree(atsru);
3511         }
3512
3513         return rc;
3514 }
3515
3516 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3517 {
3518         int i;
3519         struct pci_bus *bus;
3520         struct acpi_dmar_atsr *atsr;
3521         struct dmar_atsr_unit *atsru;
3522
3523         dev = pci_physfn(dev);
3524
3525         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3526                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3527                 if (atsr->segment == pci_domain_nr(dev->bus))
3528                         goto found;
3529         }
3530
3531         return 0;
3532
3533 found:
3534         for (bus = dev->bus; bus; bus = bus->parent) {
3535                 struct pci_dev *bridge = bus->self;
3536
3537                 if (!bridge || !pci_is_pcie(bridge) ||
3538                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
3539                         return 0;
3540
3541                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
3542                         for (i = 0; i < atsru->devices_cnt; i++)
3543                                 if (atsru->devices[i] == bridge)
3544                                         return 1;
3545                         break;
3546                 }
3547         }
3548
3549         if (atsru->include_all)
3550                 return 1;
3551
3552         return 0;
3553 }
3554
3555 int __init dmar_parse_rmrr_atsr_dev(void)
3556 {
3557         struct dmar_rmrr_unit *rmrr, *rmrr_n;
3558         struct dmar_atsr_unit *atsr, *atsr_n;
3559         int ret = 0;
3560
3561         list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3562                 ret = rmrr_parse_dev(rmrr);
3563                 if (ret)
3564                         return ret;
3565         }
3566
3567         list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3568                 ret = atsr_parse_dev(atsr);
3569                 if (ret)
3570                         return ret;
3571         }
3572
3573         return ret;
3574 }
3575
3576 /*
3577  * Here we only respond to action of unbound device from driver.
3578  *
3579  * Added device is not attached to its DMAR domain here yet. That will happen
3580  * when mapping the device to iova.
3581  */
3582 static int device_notifier(struct notifier_block *nb,
3583                                   unsigned long action, void *data)
3584 {
3585         struct device *dev = data;
3586         struct pci_dev *pdev = to_pci_dev(dev);
3587         struct dmar_domain *domain;
3588
3589         if (iommu_no_mapping(dev))
3590                 return 0;
3591
3592         domain = find_domain(pdev);
3593         if (!domain)
3594                 return 0;
3595
3596         if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3597                 domain_remove_one_dev_info(domain, pdev);
3598
3599                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3600                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3601                     list_empty(&domain->devices))
3602                         domain_exit(domain);
3603         }
3604
3605         return 0;
3606 }
3607
3608 static struct notifier_block device_nb = {
3609         .notifier_call = device_notifier,
3610 };
3611
3612 int __init intel_iommu_init(void)
3613 {
3614         int ret = 0;
3615
3616         /* VT-d is required for a TXT/tboot launch, so enforce that */
3617         force_on = tboot_force_iommu();
3618
3619         if (dmar_table_init()) {
3620                 if (force_on)
3621                         panic("tboot: Failed to initialize DMAR table\n");
3622                 return  -ENODEV;
3623         }
3624
3625         if (dmar_dev_scope_init() < 0) {
3626                 if (force_on)
3627                         panic("tboot: Failed to initialize DMAR device scope\n");
3628                 return  -ENODEV;
3629         }
3630
3631         if (no_iommu || dmar_disabled)
3632                 return -ENODEV;
3633
3634         if (iommu_init_mempool()) {
3635                 if (force_on)
3636                         panic("tboot: Failed to initialize iommu memory\n");
3637                 return  -ENODEV;
3638         }
3639
3640         if (list_empty(&dmar_rmrr_units))
3641                 printk(KERN_INFO "DMAR: No RMRR found\n");
3642
3643         if (list_empty(&dmar_atsr_units))
3644                 printk(KERN_INFO "DMAR: No ATSR found\n");
3645
3646         if (dmar_init_reserved_ranges()) {
3647                 if (force_on)
3648                         panic("tboot: Failed to reserve iommu ranges\n");
3649                 return  -ENODEV;
3650         }
3651
3652         init_no_remapping_devices();
3653
3654         ret = init_dmars();
3655         if (ret) {
3656                 if (force_on)
3657                         panic("tboot: Failed to initialize DMARs\n");
3658                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3659                 put_iova_domain(&reserved_iova_list);
3660                 iommu_exit_mempool();
3661                 return ret;
3662         }
3663         printk(KERN_INFO
3664         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3665
3666         init_timer(&unmap_timer);
3667 #ifdef CONFIG_SWIOTLB
3668         swiotlb = 0;
3669 #endif
3670         dma_ops = &intel_dma_ops;
3671
3672         init_iommu_pm_ops();
3673
3674         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
3675
3676         bus_register_notifier(&pci_bus_type, &device_nb);
3677
3678         intel_iommu_enabled = 1;
3679
3680         return 0;
3681 }
3682
3683 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3684                                            struct pci_dev *pdev)
3685 {
3686         struct pci_dev *tmp, *parent;
3687
3688         if (!iommu || !pdev)
3689                 return;
3690
3691         /* dependent device detach */
3692         tmp = pci_find_upstream_pcie_bridge(pdev);
3693         /* Secondary interface's bus number and devfn 0 */
3694         if (tmp) {
3695                 parent = pdev->bus->self;
3696                 while (parent != tmp) {
3697                         iommu_detach_dev(iommu, parent->bus->number,
3698                                          parent->devfn);
3699                         parent = parent->bus->self;
3700                 }
3701                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3702                         iommu_detach_dev(iommu,
3703                                 tmp->subordinate->number, 0);
3704                 else /* this is a legacy PCI bridge */
3705                         iommu_detach_dev(iommu, tmp->bus->number,
3706                                          tmp->devfn);
3707         }
3708 }
3709
3710 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3711                                           struct pci_dev *pdev)
3712 {
3713         struct device_domain_info *info;
3714         struct intel_iommu *iommu;
3715         unsigned long flags;
3716         int found = 0;
3717         struct list_head *entry, *tmp;
3718
3719         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3720                                 pdev->devfn);
3721         if (!iommu)
3722                 return;
3723
3724         spin_lock_irqsave(&device_domain_lock, flags);
3725         list_for_each_safe(entry, tmp, &domain->devices) {
3726                 info = list_entry(entry, struct device_domain_info, link);
3727                 if (info->segment == pci_domain_nr(pdev->bus) &&
3728                     info->bus == pdev->bus->number &&
3729                     info->devfn == pdev->devfn) {
3730                         list_del(&info->link);
3731                         list_del(&info->global);
3732                         if (info->dev)
3733                                 info->dev->dev.archdata.iommu = NULL;
3734                         spin_unlock_irqrestore(&device_domain_lock, flags);
3735
3736                         iommu_disable_dev_iotlb(info);
3737                         iommu_detach_dev(iommu, info->bus, info->devfn);
3738                         iommu_detach_dependent_devices(iommu, pdev);
3739                         free_devinfo_mem(info);
3740
3741                         spin_lock_irqsave(&device_domain_lock, flags);
3742
3743                         if (found)
3744                                 break;
3745                         else
3746                                 continue;
3747                 }
3748
3749                 /* if there is no other devices under the same iommu
3750                  * owned by this domain, clear this iommu in iommu_bmp
3751                  * update iommu count and coherency
3752                  */
3753                 if (iommu == device_to_iommu(info->segment, info->bus,
3754                                             info->devfn))
3755                         found = 1;
3756         }
3757
3758         spin_unlock_irqrestore(&device_domain_lock, flags);
3759
3760         if (found == 0) {
3761                 unsigned long tmp_flags;
3762                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3763                 clear_bit(iommu->seq_id, domain->iommu_bmp);
3764                 domain->iommu_count--;
3765                 domain_update_iommu_cap(domain);
3766                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3767
3768                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3769                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3770                         spin_lock_irqsave(&iommu->lock, tmp_flags);
3771                         clear_bit(domain->id, iommu->domain_ids);
3772                         iommu->domains[domain->id] = NULL;
3773                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3774                 }
3775         }
3776 }
3777
3778 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3779 {
3780         struct device_domain_info *info;
3781         struct intel_iommu *iommu;
3782         unsigned long flags1, flags2;
3783
3784         spin_lock_irqsave(&device_domain_lock, flags1);
3785         while (!list_empty(&domain->devices)) {
3786                 info = list_entry(domain->devices.next,
3787                         struct device_domain_info, link);
3788                 list_del(&info->link);
3789                 list_del(&info->global);
3790                 if (info->dev)
3791                         info->dev->dev.archdata.iommu = NULL;
3792
3793                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3794
3795                 iommu_disable_dev_iotlb(info);
3796                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3797                 iommu_detach_dev(iommu, info->bus, info->devfn);
3798                 iommu_detach_dependent_devices(iommu, info->dev);
3799
3800                 /* clear this iommu in iommu_bmp, update iommu count
3801                  * and capabilities
3802                  */
3803                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3804                 if (test_and_clear_bit(iommu->seq_id,
3805                                        domain->iommu_bmp)) {
3806                         domain->iommu_count--;
3807                         domain_update_iommu_cap(domain);
3808                 }
3809                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3810
3811                 free_devinfo_mem(info);
3812                 spin_lock_irqsave(&device_domain_lock, flags1);
3813         }
3814         spin_unlock_irqrestore(&device_domain_lock, flags1);
3815 }
3816
3817 /* domain id for virtual machine, it won't be set in context */
3818 static unsigned long vm_domid;
3819
3820 static struct dmar_domain *iommu_alloc_vm_domain(void)
3821 {
3822         struct dmar_domain *domain;
3823
3824         domain = alloc_domain_mem();
3825         if (!domain)
3826                 return NULL;
3827
3828         domain->id = vm_domid++;
3829         domain->nid = -1;
3830         memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
3831         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3832
3833         return domain;
3834 }
3835
3836 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3837 {
3838         int adjust_width;
3839
3840         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3841         spin_lock_init(&domain->iommu_lock);
3842
3843         domain_reserve_special_ranges(domain);
3844
3845         /* calculate AGAW */
3846         domain->gaw = guest_width;
3847         adjust_width = guestwidth_to_adjustwidth(guest_width);
3848         domain->agaw = width_to_agaw(adjust_width);
3849
3850         INIT_LIST_HEAD(&domain->devices);
3851
3852         domain->iommu_count = 0;
3853         domain->iommu_coherency = 0;
3854         domain->iommu_snooping = 0;
3855         domain->iommu_superpage = 0;
3856         domain->max_addr = 0;
3857         domain->nid = -1;
3858
3859         /* always allocate the top pgd */
3860         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3861         if (!domain->pgd)
3862                 return -ENOMEM;
3863         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3864         return 0;
3865 }
3866
3867 static void iommu_free_vm_domain(struct dmar_domain *domain)
3868 {
3869         unsigned long flags;
3870         struct dmar_drhd_unit *drhd;
3871         struct intel_iommu *iommu;
3872         unsigned long i;
3873         unsigned long ndomains;
3874
3875         for_each_drhd_unit(drhd) {
3876                 if (drhd->ignored)
3877                         continue;
3878                 iommu = drhd->iommu;
3879
3880                 ndomains = cap_ndoms(iommu->cap);
3881                 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3882                         if (iommu->domains[i] == domain) {
3883                                 spin_lock_irqsave(&iommu->lock, flags);
3884                                 clear_bit(i, iommu->domain_ids);
3885                                 iommu->domains[i] = NULL;
3886                                 spin_unlock_irqrestore(&iommu->lock, flags);
3887                                 break;
3888                         }
3889                 }
3890         }
3891 }
3892
3893 static void vm_domain_exit(struct dmar_domain *domain)
3894 {
3895         /* Domain 0 is reserved, so dont process it */
3896         if (!domain)
3897                 return;
3898
3899         vm_domain_remove_all_dev_info(domain);
3900         /* destroy iovas */
3901         put_iova_domain(&domain->iovad);
3902
3903         /* clear ptes */
3904         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3905
3906         /* free page tables */
3907         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3908
3909         iommu_free_vm_domain(domain);
3910         free_domain_mem(domain);
3911 }
3912
3913 static int intel_iommu_domain_init(struct iommu_domain *domain)
3914 {
3915         struct dmar_domain *dmar_domain;
3916
3917         dmar_domain = iommu_alloc_vm_domain();
3918         if (!dmar_domain) {
3919                 printk(KERN_ERR
3920                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3921                 return -ENOMEM;
3922         }
3923         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3924                 printk(KERN_ERR
3925                         "intel_iommu_domain_init() failed\n");
3926                 vm_domain_exit(dmar_domain);
3927                 return -ENOMEM;
3928         }
3929         domain_update_iommu_cap(dmar_domain);
3930         domain->priv = dmar_domain;
3931
3932         return 0;
3933 }
3934
3935 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3936 {
3937         struct dmar_domain *dmar_domain = domain->priv;
3938
3939         domain->priv = NULL;
3940         vm_domain_exit(dmar_domain);
3941 }
3942
3943 static int intel_iommu_attach_device(struct iommu_domain *domain,
3944                                      struct device *dev)
3945 {
3946         struct dmar_domain *dmar_domain = domain->priv;
3947         struct pci_dev *pdev = to_pci_dev(dev);
3948         struct intel_iommu *iommu;
3949         int addr_width;
3950
3951         /* normally pdev is not mapped */
3952         if (unlikely(domain_context_mapped(pdev))) {
3953                 struct dmar_domain *old_domain;
3954
3955                 old_domain = find_domain(pdev);
3956                 if (old_domain) {
3957                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3958                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3959                                 domain_remove_one_dev_info(old_domain, pdev);
3960                         else
3961                                 domain_remove_dev_info(old_domain);
3962                 }
3963         }
3964
3965         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3966                                 pdev->devfn);
3967         if (!iommu)
3968                 return -ENODEV;
3969
3970         /* check if this iommu agaw is sufficient for max mapped address */
3971         addr_width = agaw_to_width(iommu->agaw);
3972         if (addr_width > cap_mgaw(iommu->cap))
3973                 addr_width = cap_mgaw(iommu->cap);
3974
3975         if (dmar_domain->max_addr > (1LL << addr_width)) {
3976                 printk(KERN_ERR "%s: iommu width (%d) is not "
3977                        "sufficient for the mapped address (%llx)\n",
3978                        __func__, addr_width, dmar_domain->max_addr);
3979                 return -EFAULT;
3980         }
3981         dmar_domain->gaw = addr_width;
3982
3983         /*
3984          * Knock out extra levels of page tables if necessary
3985          */
3986         while (iommu->agaw < dmar_domain->agaw) {
3987                 struct dma_pte *pte;
3988
3989                 pte = dmar_domain->pgd;
3990                 if (dma_pte_present(pte)) {
3991                         dmar_domain->pgd = (struct dma_pte *)
3992                                 phys_to_virt(dma_pte_addr(pte));
3993                         free_pgtable_page(pte);
3994                 }
3995                 dmar_domain->agaw--;
3996         }
3997
3998         return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3999 }
4000
4001 static void intel_iommu_detach_device(struct iommu_domain *domain,
4002                                       struct device *dev)
4003 {
4004         struct dmar_domain *dmar_domain = domain->priv;
4005         struct pci_dev *pdev = to_pci_dev(dev);
4006
4007         domain_remove_one_dev_info(dmar_domain, pdev);
4008 }
4009
4010 static int intel_iommu_map(struct iommu_domain *domain,
4011                            unsigned long iova, phys_addr_t hpa,
4012                            size_t size, int iommu_prot)
4013 {
4014         struct dmar_domain *dmar_domain = domain->priv;
4015         u64 max_addr;
4016         int prot = 0;
4017         int ret;
4018
4019         if (iommu_prot & IOMMU_READ)
4020                 prot |= DMA_PTE_READ;
4021         if (iommu_prot & IOMMU_WRITE)
4022                 prot |= DMA_PTE_WRITE;
4023         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4024                 prot |= DMA_PTE_SNP;
4025
4026         max_addr = iova + size;
4027         if (dmar_domain->max_addr < max_addr) {
4028                 u64 end;
4029
4030                 /* check if minimum agaw is sufficient for mapped address */
4031                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4032                 if (end < max_addr) {
4033                         printk(KERN_ERR "%s: iommu width (%d) is not "
4034                                "sufficient for the mapped address (%llx)\n",
4035                                __func__, dmar_domain->gaw, max_addr);
4036                         return -EFAULT;
4037                 }
4038                 dmar_domain->max_addr = max_addr;
4039         }
4040         /* Round up size to next multiple of PAGE_SIZE, if it and
4041            the low bits of hpa would take us onto the next page */
4042         size = aligned_nrpages(hpa, size);
4043         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4044                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4045         return ret;
4046 }
4047
4048 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4049                              unsigned long iova, size_t size)
4050 {
4051         struct dmar_domain *dmar_domain = domain->priv;
4052         int order;
4053
4054         order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
4055                             (iova + size - 1) >> VTD_PAGE_SHIFT);
4056
4057         if (dmar_domain->max_addr == iova + size)
4058                 dmar_domain->max_addr = iova;
4059
4060         return PAGE_SIZE << order;
4061 }
4062
4063 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4064                                             unsigned long iova)
4065 {
4066         struct dmar_domain *dmar_domain = domain->priv;
4067         struct dma_pte *pte;
4068         u64 phys = 0;
4069
4070         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
4071         if (pte)
4072                 phys = dma_pte_addr(pte);
4073
4074         return phys;
4075 }
4076
4077 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4078                                       unsigned long cap)
4079 {
4080         struct dmar_domain *dmar_domain = domain->priv;
4081
4082         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4083                 return dmar_domain->iommu_snooping;
4084         if (cap == IOMMU_CAP_INTR_REMAP)
4085                 return intr_remapping_enabled;
4086
4087         return 0;
4088 }
4089
4090 /*
4091  * Group numbers are arbitrary.  Device with the same group number
4092  * indicate the iommu cannot differentiate between them.  To avoid
4093  * tracking used groups we just use the seg|bus|devfn of the lowest
4094  * level we're able to differentiate devices
4095  */
4096 static int intel_iommu_device_group(struct device *dev, unsigned int *groupid)
4097 {
4098         struct pci_dev *pdev = to_pci_dev(dev);
4099         struct pci_dev *bridge;
4100         union {
4101                 struct {
4102                         u8 devfn;
4103                         u8 bus;
4104                         u16 segment;
4105                 } pci;
4106                 u32 group;
4107         } id;
4108
4109         if (iommu_no_mapping(dev))
4110                 return -ENODEV;
4111
4112         id.pci.segment = pci_domain_nr(pdev->bus);
4113         id.pci.bus = pdev->bus->number;
4114         id.pci.devfn = pdev->devfn;
4115
4116         if (!device_to_iommu(id.pci.segment, id.pci.bus, id.pci.devfn))
4117                 return -ENODEV;
4118
4119         bridge = pci_find_upstream_pcie_bridge(pdev);
4120         if (bridge) {
4121                 if (pci_is_pcie(bridge)) {
4122                         id.pci.bus = bridge->subordinate->number;
4123                         id.pci.devfn = 0;
4124                 } else {
4125                         id.pci.bus = bridge->bus->number;
4126                         id.pci.devfn = bridge->devfn;
4127                 }
4128         }
4129
4130         if (!pdev->is_virtfn && iommu_group_mf)
4131                 id.pci.devfn = PCI_DEVFN(PCI_SLOT(id.pci.devfn), 0);
4132
4133         *groupid = id.group;
4134
4135         return 0;
4136 }
4137
4138 static struct iommu_ops intel_iommu_ops = {
4139         .domain_init    = intel_iommu_domain_init,
4140         .domain_destroy = intel_iommu_domain_destroy,
4141         .attach_dev     = intel_iommu_attach_device,
4142         .detach_dev     = intel_iommu_detach_device,
4143         .map            = intel_iommu_map,
4144         .unmap          = intel_iommu_unmap,
4145         .iova_to_phys   = intel_iommu_iova_to_phys,
4146         .domain_has_cap = intel_iommu_domain_has_cap,
4147         .device_group   = intel_iommu_device_group,
4148         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4149 };
4150
4151 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
4152 {
4153         /*
4154          * Mobile 4 Series Chipset neglects to set RWBF capability,
4155          * but needs it:
4156          */
4157         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4158         rwbf_quirk = 1;
4159
4160         /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
4161         if (dev->revision == 0x07) {
4162                 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4163                 dmar_map_gfx = 0;
4164         }
4165 }
4166
4167 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4168
4169 #define GGC 0x52
4170 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4171 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4172 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4173 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4174 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4175 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4176 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4177 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4178
4179 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4180 {
4181         unsigned short ggc;
4182
4183         if (pci_read_config_word(dev, GGC, &ggc))
4184                 return;
4185
4186         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4187                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4188                 dmar_map_gfx = 0;
4189         } else if (dmar_map_gfx) {
4190                 /* we have to ensure the gfx device is idle before we flush */
4191                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4192                 intel_iommu_strict = 1;
4193        }
4194 }
4195 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4196 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4197 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4198 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4199
4200 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4201    ISOCH DMAR unit for the Azalia sound device, but not give it any
4202    TLB entries, which causes it to deadlock. Check for that.  We do
4203    this in a function called from init_dmars(), instead of in a PCI
4204    quirk, because we don't want to print the obnoxious "BIOS broken"
4205    message if VT-d is actually disabled.
4206 */
4207 static void __init check_tylersburg_isoch(void)
4208 {
4209         struct pci_dev *pdev;
4210         uint32_t vtisochctrl;
4211
4212         /* If there's no Azalia in the system anyway, forget it. */
4213         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4214         if (!pdev)
4215                 return;
4216         pci_dev_put(pdev);
4217
4218         /* System Management Registers. Might be hidden, in which case
4219            we can't do the sanity check. But that's OK, because the
4220            known-broken BIOSes _don't_ actually hide it, so far. */
4221         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4222         if (!pdev)
4223                 return;
4224
4225         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4226                 pci_dev_put(pdev);
4227                 return;
4228         }
4229
4230         pci_dev_put(pdev);
4231
4232         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4233         if (vtisochctrl & 1)
4234                 return;
4235
4236         /* Drop all bits other than the number of TLB entries */
4237         vtisochctrl &= 0x1c;
4238
4239         /* If we have the recommended number of TLB entries (16), fine. */
4240         if (vtisochctrl == 0x10)
4241                 return;
4242
4243         /* Zero TLB entries? You get to ride the short bus to school. */
4244         if (!vtisochctrl) {
4245                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4246                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4247                      dmi_get_system_info(DMI_BIOS_VENDOR),
4248                      dmi_get_system_info(DMI_BIOS_VERSION),
4249                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4250                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4251                 return;
4252         }
4253
4254         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4255                vtisochctrl);
4256 }