linux/drivers/iommu/amd_iommu.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
   3 * Author: Joerg Roedel <joerg.roedel@amd.com>
   4 *         Leo Duran <leo.duran@amd.com>
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License version 2 as published
   8 * by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope that it will be useful,
  11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 * GNU General Public License for more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program; if not, write to the Free Software
  17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18 */
  19
  20#include <linux/ratelimit.h>
  21#include <linux/pci.h>
  22#include <linux/pci-ats.h>
  23#include <linux/bitmap.h>
  24#include <linux/slab.h>
  25#include <linux/debugfs.h>
  26#include <linux/scatterlist.h>
  27#include <linux/dma-mapping.h>
  28#include <linux/iommu-helper.h>
  29#include <linux/iommu.h>
  30#include <linux/delay.h>
  31#include <linux/amd-iommu.h>
  32#include <linux/notifier.h>
  33#include <linux/export.h>
  34#include <linux/irq.h>
  35#include <linux/msi.h>
  36#include <asm/irq_remapping.h>
  37#include <asm/io_apic.h>
  38#include <asm/apic.h>
  39#include <asm/hw_irq.h>
  40#include <asm/msidef.h>
  41#include <asm/proto.h>
  42#include <asm/iommu.h>
  43#include <asm/gart.h>
  44#include <asm/dma.h>
  45
  46#include "amd_iommu_proto.h"
  47#include "amd_iommu_types.h"
  48#include "irq_remapping.h"
  49#include "pci.h"
  50
  51#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
  52
  53#define LOOP_TIMEOUT    100000
  54
  55/*
  56 * This bitmap is used to advertise the page sizes our hardware support
  57 * to the IOMMU core, which will then use this information to split
  58 * physically contiguous memory regions it is mapping into page sizes
  59 * that we support.
  60 *
  61 * 512GB Pages are not supported due to a hardware bug
  62 */
  63#define AMD_IOMMU_PGSIZES       ((~0xFFFUL) & ~(2ULL << 38))
  64
  65static DEFINE_RWLOCK(amd_iommu_devtable_lock);
  66
  67/* A list of preallocated protection domains */
  68static LIST_HEAD(iommu_pd_list);
  69static DEFINE_SPINLOCK(iommu_pd_list_lock);
  70
  71/* List of all available dev_data structures */
  72static LIST_HEAD(dev_data_list);
  73static DEFINE_SPINLOCK(dev_data_list_lock);
  74
  75LIST_HEAD(ioapic_map);
  76LIST_HEAD(hpet_map);
  77
  78/*
  79 * Domain for untranslated devices - only allocated
  80 * if iommu=pt passed on kernel cmd line.
  81 */
  82static struct protection_domain *pt_domain;
  83
  84static struct iommu_ops amd_iommu_ops;
  85
  86static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
  87int amd_iommu_max_glx_val = -1;
  88
  89static struct dma_map_ops amd_iommu_dma_ops;
  90
  91/*
  92 * general struct to manage commands send to an IOMMU
  93 */
  94struct iommu_cmd {
  95        u32 data[4];
  96};
  97
  98struct kmem_cache *amd_iommu_irq_cache;
  99
 100static void update_domain(struct protection_domain *domain);
 101static int __init alloc_passthrough_domain(void);
 102
 103/****************************************************************************
 104 *
 105 * Helper functions
 106 *
 107 ****************************************************************************/
 108
 109static struct iommu_dev_data *alloc_dev_data(u16 devid)
 110{
 111        struct iommu_dev_data *dev_data;
 112        unsigned long flags;
 113
 114        dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
 115        if (!dev_data)
 116                return NULL;
 117
 118        dev_data->devid = devid;
 119        atomic_set(&dev_data->bind, 0);
 120
 121        spin_lock_irqsave(&dev_data_list_lock, flags);
 122        list_add_tail(&dev_data->dev_data_list, &dev_data_list);
 123        spin_unlock_irqrestore(&dev_data_list_lock, flags);
 124
 125        return dev_data;
 126}
 127
 128static void free_dev_data(struct iommu_dev_data *dev_data)
 129{
 130        unsigned long flags;
 131
 132        spin_lock_irqsave(&dev_data_list_lock, flags);
 133        list_del(&dev_data->dev_data_list);
 134        spin_unlock_irqrestore(&dev_data_list_lock, flags);
 135
 136        if (dev_data->group)
 137                iommu_group_put(dev_data->group);
 138
 139        kfree(dev_data);
 140}
 141
 142static struct iommu_dev_data *search_dev_data(u16 devid)
 143{
 144        struct iommu_dev_data *dev_data;
 145        unsigned long flags;
 146
 147        spin_lock_irqsave(&dev_data_list_lock, flags);
 148        list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
 149                if (dev_data->devid == devid)
 150                        goto out_unlock;
 151        }
 152
 153        dev_data = NULL;
 154
 155out_unlock:
 156        spin_unlock_irqrestore(&dev_data_list_lock, flags);
 157
 158        return dev_data;
 159}
 160
 161static struct iommu_dev_data *find_dev_data(u16 devid)
 162{
 163        struct iommu_dev_data *dev_data;
 164
 165        dev_data = search_dev_data(devid);
 166
 167        if (dev_data == NULL)
 168                dev_data = alloc_dev_data(devid);
 169
 170        return dev_data;
 171}
 172
 173static inline u16 get_device_id(struct device *dev)
 174{
 175        struct pci_dev *pdev = to_pci_dev(dev);
 176
 177        return PCI_DEVID(pdev->bus->number, pdev->devfn);
 178}
 179
 180static struct iommu_dev_data *get_dev_data(struct device *dev)
 181{
 182        return dev->archdata.iommu;
 183}
 184
 185static bool pci_iommuv2_capable(struct pci_dev *pdev)
 186{
 187        static const int caps[] = {
 188                PCI_EXT_CAP_ID_ATS,
 189                PCI_EXT_CAP_ID_PRI,
 190                PCI_EXT_CAP_ID_PASID,
 191        };
 192        int i, pos;
 193
 194        for (i = 0; i < 3; ++i) {
 195                pos = pci_find_ext_capability(pdev, caps[i]);
 196                if (pos == 0)
 197                        return false;
 198        }
 199
 200        return true;
 201}
 202
 203static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
 204{
 205        struct iommu_dev_data *dev_data;
 206
 207        dev_data = get_dev_data(&pdev->dev);
 208
 209        return dev_data->errata & (1 << erratum) ? true : false;
 210}
 211
 212/*
 213 * In this function the list of preallocated protection domains is traversed to
 214 * find the domain for a specific device
 215 */
 216static struct dma_ops_domain *find_protection_domain(u16 devid)
 217{
 218        struct dma_ops_domain *entry, *ret = NULL;
 219        unsigned long flags;
 220        u16 alias = amd_iommu_alias_table[devid];
 221
 222        if (list_empty(&iommu_pd_list))
 223                return NULL;
 224
 225        spin_lock_irqsave(&iommu_pd_list_lock, flags);
 226
 227        list_for_each_entry(entry, &iommu_pd_list, list) {
 228                if (entry->target_dev == devid ||
 229                    entry->target_dev == alias) {
 230                        ret = entry;
 231                        break;
 232                }
 233        }
 234
 235        spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
 236
 237        return ret;
 238}
 239
 240/*
 241 * This function checks if the driver got a valid device from the caller to
 242 * avoid dereferencing invalid pointers.
 243 */
 244static bool check_device(struct device *dev)
 245{
 246        u16 devid;
 247
 248        if (!dev || !dev->dma_mask)
 249                return false;
 250
 251        /* No device or no PCI device */
 252        if (dev->bus != &pci_bus_type)
 253                return false;
 254
 255        devid = get_device_id(dev);
 256
 257        /* Out of our scope? */
 258        if (devid > amd_iommu_last_bdf)
 259                return false;
 260
 261        if (amd_iommu_rlookup_table[devid] == NULL)
 262                return false;
 263
 264        return true;
 265}
 266
 267static struct pci_bus *find_hosted_bus(struct pci_bus *bus)
 268{
 269        while (!bus->self) {
 270                if (!pci_is_root_bus(bus))
 271                        bus = bus->parent;
 272                else
 273                        return ERR_PTR(-ENODEV);
 274        }
 275
 276        return bus;
 277}
 278
 279#define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
 280
 281static struct pci_dev *get_isolation_root(struct pci_dev *pdev)
 282{
 283        struct pci_dev *dma_pdev = pdev;
 284
 285        /* Account for quirked devices */
 286        swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
 287
 288        /*
 289         * If it's a multifunction device that does not support our
 290         * required ACS flags, add to the same group as lowest numbered
 291         * function that also does not suport the required ACS flags.
 292         */
 293        if (dma_pdev->multifunction &&
 294            !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
 295                u8 i, slot = PCI_SLOT(dma_pdev->devfn);
 296
 297                for (i = 0; i < 8; i++) {
 298                        struct pci_dev *tmp;
 299
 300                        tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
 301                        if (!tmp)
 302                                continue;
 303
 304                        if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
 305                                swap_pci_ref(&dma_pdev, tmp);
 306                                break;
 307                        }
 308                        pci_dev_put(tmp);
 309                }
 310        }
 311
 312        /*
 313         * Devices on the root bus go through the iommu.  If that's not us,
 314         * find the next upstream device and test ACS up to the root bus.
 315         * Finding the next device may require skipping virtual buses.
 316         */
 317        while (!pci_is_root_bus(dma_pdev->bus)) {
 318                struct pci_bus *bus = find_hosted_bus(dma_pdev->bus);
 319                if (IS_ERR(bus))
 320                        break;
 321
 322                if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
 323                        break;
 324
 325                swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
 326        }
 327
 328        return dma_pdev;
 329}
 330
 331static int use_pdev_iommu_group(struct pci_dev *pdev, struct device *dev)
 332{
 333        struct iommu_group *group = iommu_group_get(&pdev->dev);
 334        int ret;
 335
 336        if (!group) {
 337                group = iommu_group_alloc();
 338                if (IS_ERR(group))
 339                        return PTR_ERR(group);
 340
 341                WARN_ON(&pdev->dev != dev);
 342        }
 343
 344        ret = iommu_group_add_device(group, dev);
 345        iommu_group_put(group);
 346        return ret;
 347}
 348
 349static int use_dev_data_iommu_group(struct iommu_dev_data *dev_data,
 350                                    struct device *dev)
 351{
 352        if (!dev_data->group) {
 353                struct iommu_group *group = iommu_group_alloc();
 354                if (IS_ERR(group))
 355                        return PTR_ERR(group);
 356
 357                dev_data->group = group;
 358        }
 359
 360        return iommu_group_add_device(dev_data->group, dev);
 361}
 362
 363static int init_iommu_group(struct device *dev)
 364{
 365        struct iommu_dev_data *dev_data;
 366        struct iommu_group *group;
 367        struct pci_dev *dma_pdev;
 368        int ret;
 369
 370        group = iommu_group_get(dev);
 371        if (group) {
 372                iommu_group_put(group);
 373                return 0;
 374        }
 375
 376        dev_data = find_dev_data(get_device_id(dev));
 377        if (!dev_data)
 378                return -ENOMEM;
 379
 380        if (dev_data->alias_data) {
 381                u16 alias;
 382                struct pci_bus *bus;
 383
 384                if (dev_data->alias_data->group)
 385                        goto use_group;
 386
 387                /*
 388                 * If the alias device exists, it's effectively just a first
 389                 * level quirk for finding the DMA source.
 390                 */
 391                alias = amd_iommu_alias_table[dev_data->devid];
 392                dma_pdev = pci_get_bus_and_slot(alias >> 8, alias & 0xff);
 393                if (dma_pdev) {
 394                        dma_pdev = get_isolation_root(dma_pdev);
 395                        goto use_pdev;
 396                }
 397
 398                /*
 399                 * If the alias is virtual, try to find a parent device
 400                 * and test whether the IOMMU group is actualy rooted above
 401                 * the alias.  Be careful to also test the parent device if
 402                 * we think the alias is the root of the group.
 403                 */
 404                bus = pci_find_bus(0, alias >> 8);
 405                if (!bus)
 406                        goto use_group;
 407
 408                bus = find_hosted_bus(bus);
 409                if (IS_ERR(bus) || !bus->self)
 410                        goto use_group;
 411
 412                dma_pdev = get_isolation_root(pci_dev_get(bus->self));
 413                if (dma_pdev != bus->self || (dma_pdev->multifunction &&
 414                    !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)))
 415                        goto use_pdev;
 416
 417                pci_dev_put(dma_pdev);
 418                goto use_group;
 419        }
 420
 421        dma_pdev = get_isolation_root(pci_dev_get(to_pci_dev(dev)));
 422use_pdev:
 423        ret = use_pdev_iommu_group(dma_pdev, dev);
 424        pci_dev_put(dma_pdev);
 425        return ret;
 426use_group:
 427        return use_dev_data_iommu_group(dev_data->alias_data, dev);
 428}
 429
 430static int iommu_init_device(struct device *dev)
 431{
 432        struct pci_dev *pdev = to_pci_dev(dev);
 433        struct iommu_dev_data *dev_data;
 434        u16 alias;
 435        int ret;
 436
 437        if (dev->archdata.iommu)
 438                return 0;
 439
 440        dev_data = find_dev_data(get_device_id(dev));
 441        if (!dev_data)
 442                return -ENOMEM;
 443
 444        alias = amd_iommu_alias_table[dev_data->devid];
 445        if (alias != dev_data->devid) {
 446                struct iommu_dev_data *alias_data;
 447
 448                alias_data = find_dev_data(alias);
 449                if (alias_data == NULL) {
 450                        pr_err("AMD-Vi: Warning: Unhandled device %s\n",
 451                                        dev_name(dev));
 452                        free_dev_data(dev_data);
 453                        return -ENOTSUPP;
 454                }
 455                dev_data->alias_data = alias_data;
 456        }
 457
 458        ret = init_iommu_group(dev);
 459        if (ret) {
 460                free_dev_data(dev_data);
 461                return ret;
 462        }
 463
 464        if (pci_iommuv2_capable(pdev)) {
 465                struct amd_iommu *iommu;
 466
 467                iommu              = amd_iommu_rlookup_table[dev_data->devid];
 468                dev_data->iommu_v2 = iommu->is_iommu_v2;
 469        }
 470
 471        dev->archdata.iommu = dev_data;
 472
 473        return 0;
 474}
 475
 476static void iommu_ignore_device(struct device *dev)
 477{
 478        u16 devid, alias;
 479
 480        devid = get_device_id(dev);
 481        alias = amd_iommu_alias_table[devid];
 482
 483        memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
 484        memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
 485
 486        amd_iommu_rlookup_table[devid] = NULL;
 487        amd_iommu_rlookup_table[alias] = NULL;
 488}
 489
 490static void iommu_uninit_device(struct device *dev)
 491{
 492        iommu_group_remove_device(dev);
 493
 494        /*
 495         * Nothing to do here - we keep dev_data around for unplugged devices
 496         * and reuse it when the device is re-plugged - not doing so would
 497         * introduce a ton of races.
 498         */
 499}
 500
 501void __init amd_iommu_uninit_devices(void)
 502{
 503        struct iommu_dev_data *dev_data, *n;
 504        struct pci_dev *pdev = NULL;
 505
 506        for_each_pci_dev(pdev) {
 507
 508                if (!check_device(&pdev->dev))
 509                        continue;
 510
 511                iommu_uninit_device(&pdev->dev);
 512        }
 513
 514        /* Free all of our dev_data structures */
 515        list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
 516                free_dev_data(dev_data);
 517}
 518
 519int __init amd_iommu_init_devices(void)
 520{
 521        struct pci_dev *pdev = NULL;
 522        int ret = 0;
 523
 524        for_each_pci_dev(pdev) {
 525
 526                if (!check_device(&pdev->dev))
 527                        continue;
 528
 529                ret = iommu_init_device(&pdev->dev);
 530                if (ret == -ENOTSUPP)
 531                        iommu_ignore_device(&pdev->dev);
 532                else if (ret)
 533                        goto out_free;
 534        }
 535
 536        return 0;
 537
 538out_free:
 539
 540        amd_iommu_uninit_devices();
 541
 542        return ret;
 543}
 544#ifdef CONFIG_AMD_IOMMU_STATS
 545
 546/*
 547 * Initialization code for statistics collection
 548 */
 549
 550DECLARE_STATS_COUNTER(compl_wait);
 551DECLARE_STATS_COUNTER(cnt_map_single);
 552DECLARE_STATS_COUNTER(cnt_unmap_single);
 553DECLARE_STATS_COUNTER(cnt_map_sg);
 554DECLARE_STATS_COUNTER(cnt_unmap_sg);
 555DECLARE_STATS_COUNTER(cnt_alloc_coherent);
 556DECLARE_STATS_COUNTER(cnt_free_coherent);
 557DECLARE_STATS_COUNTER(cross_page);
 558DECLARE_STATS_COUNTER(domain_flush_single);
 559DECLARE_STATS_COUNTER(domain_flush_all);
 560DECLARE_STATS_COUNTER(alloced_io_mem);
 561DECLARE_STATS_COUNTER(total_map_requests);
 562DECLARE_STATS_COUNTER(complete_ppr);
 563DECLARE_STATS_COUNTER(invalidate_iotlb);
 564DECLARE_STATS_COUNTER(invalidate_iotlb_all);
 565DECLARE_STATS_COUNTER(pri_requests);
 566
 567static struct dentry *stats_dir;
 568static struct dentry *de_fflush;
 569
 570static void amd_iommu_stats_add(struct __iommu_counter *cnt)
 571{
 572        if (stats_dir == NULL)
 573                return;
 574
 575        cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
 576                                       &cnt->value);
 577}
 578
 579static void amd_iommu_stats_init(void)
 580{
 581        stats_dir = debugfs_create_dir("amd-iommu", NULL);
 582        if (stats_dir == NULL)
 583                return;
 584
 585        de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
 586                                         &amd_iommu_unmap_flush);
 587
 588        amd_iommu_stats_add(&compl_wait);
 589        amd_iommu_stats_add(&cnt_map_single);
 590        amd_iommu_stats_add(&cnt_unmap_single);
 591        amd_iommu_stats_add(&cnt_map_sg);
 592        amd_iommu_stats_add(&cnt_unmap_sg);
 593        amd_iommu_stats_add(&cnt_alloc_coherent);
 594        amd_iommu_stats_add(&cnt_free_coherent);
 595        amd_iommu_stats_add(&cross_page);
 596        amd_iommu_stats_add(&domain_flush_single);
 597        amd_iommu_stats_add(&domain_flush_all);
 598        amd_iommu_stats_add(&alloced_io_mem);
 599        amd_iommu_stats_add(&total_map_requests);
 600        amd_iommu_stats_add(&complete_ppr);
 601        amd_iommu_stats_add(&invalidate_iotlb);
 602        amd_iommu_stats_add(&invalidate_iotlb_all);
 603        amd_iommu_stats_add(&pri_requests);
 604}
 605
 606#endif
 607
 608/****************************************************************************
 609 *
 610 * Interrupt handling functions
 611 *
 612 ****************************************************************************/
 613
 614static void dump_dte_entry(u16 devid)
 615{
 616        int i;
 617
 618        for (i = 0; i < 4; ++i)
 619                pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
 620                        amd_iommu_dev_table[devid].data[i]);
 621}
 622
 623static void dump_command(unsigned long phys_addr)
 624{
 625        struct iommu_cmd *cmd = phys_to_virt(phys_addr);
 626        int i;
 627
 628        for (i = 0; i < 4; ++i)
 629                pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
 630}
 631
 632static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 633{
 634        int type, devid, domid, flags;
 635        volatile u32 *event = __evt;
 636        int count = 0;
 637        u64 address;
 638
 639retry:
 640        type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
 641        devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
 642        domid   = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
 643        flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
 644        address = (u64)(((u64)event[3]) << 32) | event[2];
 645
 646        if (type == 0) {
 647                /* Did we hit the erratum? */
 648                if (++count == LOOP_TIMEOUT) {
 649                        pr_err("AMD-Vi: No event written to event log\n");
 650                        return;
 651                }
 652                udelay(1);
 653                goto retry;
 654        }
 655
 656        printk(KERN_ERR "AMD-Vi: Event logged [");
 657
 658        switch (type) {
 659        case EVENT_TYPE_ILL_DEV:
 660                printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
 661                       "address=0x%016llx flags=0x%04x]\n",
 662                       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 663                       address, flags);
 664                dump_dte_entry(devid);
 665                break;
 666        case EVENT_TYPE_IO_FAULT:
 667                printk("IO_PAGE_FAULT device=%02x:%02x.%x "
 668                       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
 669                       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 670                       domid, address, flags);
 671                break;
 672        case EVENT_TYPE_DEV_TAB_ERR:
 673                printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
 674                       "address=0x%016llx flags=0x%04x]\n",
 675                       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 676                       address, flags);
 677                break;
 678        case EVENT_TYPE_PAGE_TAB_ERR:
 679                printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
 680                       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
 681                       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 682                       domid, address, flags);
 683                break;
 684        case EVENT_TYPE_ILL_CMD:
 685                printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
 686                dump_command(address);
 687                break;
 688        case EVENT_TYPE_CMD_HARD_ERR:
 689                printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
 690                       "flags=0x%04x]\n", address, flags);
 691                break;
 692        case EVENT_TYPE_IOTLB_INV_TO:
 693                printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
 694                       "address=0x%016llx]\n",
 695                       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 696                       address);
 697                break;
 698        case EVENT_TYPE_INV_DEV_REQ:
 699                printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
 700                       "address=0x%016llx flags=0x%04x]\n",
 701                       PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 702                       address, flags);
 703                break;
 704        default:
 705                printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
 706        }
 707
 708        memset(__evt, 0, 4 * sizeof(u32));
 709}
 710
 711static void iommu_poll_events(struct amd_iommu *iommu)
 712{
 713        u32 head, tail;
 714
 715        head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
 716        tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
 717
 718        while (head != tail) {
 719                iommu_print_event(iommu, iommu->evt_buf + head);
 720                head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
 721        }
 722
 723        writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
 724}
 725
 726static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
 727{
 728        struct amd_iommu_fault fault;
 729
 730        INC_STATS_COUNTER(pri_requests);
 731
 732        if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
 733                pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
 734                return;
 735        }
 736
 737        fault.address   = raw[1];
 738        fault.pasid     = PPR_PASID(raw[0]);
 739        fault.device_id = PPR_DEVID(raw[0]);
 740        fault.tag       = PPR_TAG(raw[0]);
 741        fault.flags     = PPR_FLAGS(raw[0]);
 742
 743        atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
 744}
 745
 746static void iommu_poll_ppr_log(struct amd_iommu *iommu)
 747{
 748        u32 head, tail;
 749
 750        if (iommu->ppr_log == NULL)
 751                return;
 752
 753        head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 754        tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
 755
 756        while (head != tail) {
 757                volatile u64 *raw;
 758                u64 entry[2];
 759                int i;
 760
 761                raw = (u64 *)(iommu->ppr_log + head);
 762
 763                /*
 764                 * Hardware bug: Interrupt may arrive before the entry is
 765                 * written to memory. If this happens we need to wait for the
 766                 * entry to arrive.
 767                 */
 768                for (i = 0; i < LOOP_TIMEOUT; ++i) {
 769                        if (PPR_REQ_TYPE(raw[0]) != 0)
 770                                break;
 771                        udelay(1);
 772                }
 773
 774                /* Avoid memcpy function-call overhead */
 775                entry[0] = raw[0];
 776                entry[1] = raw[1];
 777
 778                /*
 779                 * To detect the hardware bug we need to clear the entry
 780                 * back to zero.
 781                 */
 782                raw[0] = raw[1] = 0UL;
 783
 784                /* Update head pointer of hardware ring-buffer */
 785                head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
 786                writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 787
 788                /* Handle PPR entry */
 789                iommu_handle_ppr_entry(iommu, entry);
 790
 791                /* Refresh ring-buffer information */
 792                head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 793                tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
 794        }
 795}
 796
 797irqreturn_t amd_iommu_int_thread(int irq, void *data)
 798{
 799        struct amd_iommu *iommu = (struct amd_iommu *) data;
 800        u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
 801
 802        while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) {
 803                /* Enable EVT and PPR interrupts again */
 804                writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK),
 805                        iommu->mmio_base + MMIO_STATUS_OFFSET);
 806
 807                if (status & MMIO_STATUS_EVT_INT_MASK) {
 808                        pr_devel("AMD-Vi: Processing IOMMU Event Log\n");
 809                        iommu_poll_events(iommu);
 810                }
 811
 812                if (status & MMIO_STATUS_PPR_INT_MASK) {
 813                        pr_devel("AMD-Vi: Processing IOMMU PPR Log\n");
 814                        iommu_poll_ppr_log(iommu);
 815                }
 816
 817                /*
 818                 * Hardware bug: ERBT1312
 819                 * When re-enabling interrupt (by writing 1
 820                 * to clear the bit), the hardware might also try to set
 821                 * the interrupt bit in the event status register.
 822                 * In this scenario, the bit will be set, and disable
 823                 * subsequent interrupts.
 824                 *
 825                 * Workaround: The IOMMU driver should read back the
 826                 * status register and check if the interrupt bits are cleared.
 827                 * If not, driver will need to go through the interrupt handler
 828                 * again and re-clear the bits
 829                 */
 830                status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
 831        }
 832        return IRQ_HANDLED;
 833}
 834
 835irqreturn_t amd_iommu_int_handler(int irq, void *data)
 836{
 837        return IRQ_WAKE_THREAD;
 838}
 839
 840/****************************************************************************
 841 *
 842 * IOMMU command queuing functions
 843 *
 844 ****************************************************************************/
 845
 846static int wait_on_sem(volatile u64 *sem)
 847{
 848        int i = 0;
 849
 850        while (*sem == 0 && i < LOOP_TIMEOUT) {
 851                udelay(1);
 852                i += 1;
 853        }
 854
 855        if (i == LOOP_TIMEOUT) {
 856                pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
 857                return -EIO;
 858        }
 859
 860        return 0;
 861}
 862
 863static void copy_cmd_to_buffer(struct amd_iommu *iommu,
 864                               struct iommu_cmd *cmd,
 865                               u32 tail)
 866{
 867        u8 *target;
 868
 869        target = iommu->cmd_buf + tail;
 870        tail   = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
 871
 872        /* Copy command to buffer */
 873        memcpy(target, cmd, sizeof(*cmd));
 874
 875        /* Tell the IOMMU about it */
 876        writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
 877}
 878
 879static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
 880{
 881        WARN_ON(address & 0x7ULL);
 882
 883        memset(cmd, 0, sizeof(*cmd));
 884        cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
 885        cmd->data[1] = upper_32_bits(__pa(address));
 886        cmd->data[2] = 1;
 887        CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
 888}
 889
 890static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
 891{
 892        memset(cmd, 0, sizeof(*cmd));
 893        cmd->data[0] = devid;
 894        CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
 895}
 896
 897static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
 898                                  size_t size, u16 domid, int pde)
 899{
 900        u64 pages;
 901        int s;
 902
 903        pages = iommu_num_pages(address, size, PAGE_SIZE);
 904        s     = 0;
 905
 906        if (pages > 1) {
 907                /*
 908                 * If we have to flush more than one page, flush all
 909                 * TLB entries for this domain
 910                 */
 911                address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
 912                s = 1;
 913        }
 914
 915        address &= PAGE_MASK;
 916
 917        memset(cmd, 0, sizeof(*cmd));
 918        cmd->data[1] |= domid;
 919        cmd->data[2]  = lower_32_bits(address);
 920        cmd->data[3]  = upper_32_bits(address);
 921        CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
 922        if (s) /* size bit - we flush more than one 4kb page */
 923                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 924        if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
 925                cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 926}
 927
 928static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
 929                                  u64 address, size_t size)
 930{
 931        u64 pages;
 932        int s;
 933
 934        pages = iommu_num_pages(address, size, PAGE_SIZE);
 935        s     = 0;
 936
 937        if (pages > 1) {
 938                /*
 939                 * If we have to flush more than one page, flush all
 940                 * TLB entries for this domain
 941                 */
 942                address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
 943                s = 1;
 944        }
 945
 946        address &= PAGE_MASK;
 947
 948        memset(cmd, 0, sizeof(*cmd));
 949        cmd->data[0]  = devid;
 950        cmd->data[0] |= (qdep & 0xff) << 24;
 951        cmd->data[1]  = devid;
 952        cmd->data[2]  = lower_32_bits(address);
 953        cmd->data[3]  = upper_32_bits(address);
 954        CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
 955        if (s)
 956                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 957}
 958
 959static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
 960                                  u64 address, bool size)
 961{
 962        memset(cmd, 0, sizeof(*cmd));
 963
 964        address &= ~(0xfffULL);
 965
 966        cmd->data[0]  = pasid & PASID_MASK;
 967        cmd->data[1]  = domid;
 968        cmd->data[2]  = lower_32_bits(address);
 969        cmd->data[3]  = upper_32_bits(address);
 970        cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 971        cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
 972        if (size)
 973                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 974        CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
 975}
 976
 977static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
 978                                  int qdep, u64 address, bool size)
 979{
 980        memset(cmd, 0, sizeof(*cmd));
 981
 982        address &= ~(0xfffULL);
 983
 984        cmd->data[0]  = devid;
 985        cmd->data[0] |= (pasid & 0xff) << 16;
 986        cmd->data[0] |= (qdep  & 0xff) << 24;
 987        cmd->data[1]  = devid;
 988        cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
 989        cmd->data[2]  = lower_32_bits(address);
 990        cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
 991        cmd->data[3]  = upper_32_bits(address);
 992        if (size)
 993                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 994        CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
 995}
 996
 997static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
 998                               int status, int tag, bool gn)
 999{
1000        memset(cmd, 0, sizeof(*cmd));
1001
1002        cmd->data[0]  = devid;
1003        if (gn) {
1004                cmd->data[1]  = pasid & PASID_MASK;
1005                cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
1006        }
1007        cmd->data[3]  = tag & 0x1ff;
1008        cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
1009
1010        CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
1011}
1012
1013static void build_inv_all(struct iommu_cmd *cmd)
1014{
1015        memset(cmd, 0, sizeof(*cmd));
1016        CMD_SET_TYPE(cmd, CMD_INV_ALL);
1017}
1018
1019static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
1020{
1021        memset(cmd, 0, sizeof(*cmd));
1022        cmd->data[0] = devid;
1023        CMD_SET_TYPE(cmd, CMD_INV_IRT);
1024}
1025
1026/*
1027 * Writes the command to the IOMMUs command buffer and informs the
1028 * hardware about the new command.
1029 */
1030static int iommu_queue_command_sync(struct amd_iommu *iommu,
1031                                    struct iommu_cmd *cmd,
1032                                    bool sync)
1033{
1034        u32 left, tail, head, next_tail;
1035        unsigned long flags;
1036
1037        WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
1038
1039again:
1040        spin_lock_irqsave(&iommu->lock, flags);
1041
1042        head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
1043        tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
1044        next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
1045        left      = (head - next_tail) % iommu->cmd_buf_size;
1046
1047        if (left <= 2) {
1048                struct iommu_cmd sync_cmd;
1049                volatile u64 sem = 0;
1050                int ret;
1051
1052                build_completion_wait(&sync_cmd, (u64)&sem);
1053                copy_cmd_to_buffer(iommu, &sync_cmd, tail);
1054
1055                spin_unlock_irqrestore(&iommu->lock, flags);
1056
1057                if ((ret = wait_on_sem(&sem)) != 0)
1058                        return ret;
1059
1060                goto again;
1061        }
1062
1063        copy_cmd_to_buffer(iommu, cmd, tail);
1064
1065        /* We need to sync now to make sure all commands are processed */
1066        iommu->need_sync = sync;
1067
1068        spin_unlock_irqrestore(&iommu->lock, flags);
1069
1070        return 0;
1071}
1072
1073static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
1074{
1075        return iommu_queue_command_sync(iommu, cmd, true);
1076}
1077
1078/*
1079 * This function queues a completion wait command into the command
1080 * buffer of an IOMMU
1081 */
1082static int iommu_completion_wait(struct amd_iommu *iommu)
1083{
1084        struct iommu_cmd cmd;
1085        volatile u64 sem = 0;
1086        int ret;
1087
1088        if (!iommu->need_sync)
1089                return 0;
1090
1091        build_completion_wait(&cmd, (u64)&sem);
1092
1093        ret = iommu_queue_command_sync(iommu, &cmd, false);
1094        if (ret)
1095                return ret;
1096
1097        return wait_on_sem(&sem);
1098}
1099
1100static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1101{
1102        struct iommu_cmd cmd;
1103
1104        build_inv_dte(&cmd, devid);
1105
1106        return iommu_queue_command(iommu, &cmd);
1107}
1108
1109static void iommu_flush_dte_all(struct amd_iommu *iommu)
1110{
1111        u32 devid;
1112
1113        for (devid = 0; devid <= 0xffff; ++devid)
1114                iommu_flush_dte(iommu, devid);
1115
1116        iommu_completion_wait(iommu);
1117}
1118
1119/*
1120 * This function uses heavy locking and may disable irqs for some time. But
1121 * this is no issue because it is only called during resume.
1122 */
1123static void iommu_flush_tlb_all(struct amd_iommu *iommu)
1124{
1125        u32 dom_id;
1126
1127        for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
1128                struct iommu_cmd cmd;
1129                build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1130                                      dom_id, 1);
1131                iommu_queue_command(iommu, &cmd);
1132        }
1133
1134        iommu_completion_wait(iommu);
1135}
1136
1137static void iommu_flush_all(struct amd_iommu *iommu)
1138{
1139        struct iommu_cmd cmd;
1140
1141        build_inv_all(&cmd);
1142
1143        iommu_queue_command(iommu, &cmd);
1144        iommu_completion_wait(iommu);
1145}
1146
1147static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
1148{
1149        struct iommu_cmd cmd;
1150
1151        build_inv_irt(&cmd, devid);
1152
1153        iommu_queue_command(iommu, &cmd);
1154}
1155
1156static void iommu_flush_irt_all(struct amd_iommu *iommu)
1157{
1158        u32 devid;
1159
1160        for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)
1161                iommu_flush_irt(iommu, devid);
1162
1163        iommu_completion_wait(iommu);
1164}
1165
1166void iommu_flush_all_caches(struct amd_iommu *iommu)
1167{
1168        if (iommu_feature(iommu, FEATURE_IA)) {
1169                iommu_flush_all(iommu);
1170        } else {
1171                iommu_flush_dte_all(iommu);
1172                iommu_flush_irt_all(iommu);
1173                iommu_flush_tlb_all(iommu);
1174        }
1175}
1176
1177/*
1178 * Command send function for flushing on-device TLB
1179 */
1180static int device_flush_iotlb(struct iommu_dev_data *dev_data,
1181                              u64 address, size_t size)
1182{
1183        struct amd_iommu *iommu;
1184        struct iommu_cmd cmd;
1185        int qdep;
1186
1187        qdep     = dev_data->ats.qdep;
1188        iommu    = amd_iommu_rlookup_table[dev_data->devid];
1189
1190        build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1191
1192        return iommu_queue_command(iommu, &cmd);
1193}
1194
1195/*
1196 * Command send function for invalidating a device table entry
1197 */
1198static int device_flush_dte(struct iommu_dev_data *dev_data)
1199{
1200        struct amd_iommu *iommu;
1201        int ret;
1202
1203        iommu = amd_iommu_rlookup_table[dev_data->devid];
1204
1205        ret = iommu_flush_dte(iommu, dev_data->devid);
1206        if (ret)
1207                return ret;
1208
1209        if (dev_data->ats.enabled)
1210                ret = device_flush_iotlb(dev_data, 0, ~0UL);
1211
1212        return ret;
1213}
1214
1215/*
1216 * TLB invalidation function which is called from the mapping functions.
1217 * It invalidates a single PTE if the range to flush is within a single
1218 * page. Otherwise it flushes the whole TLB of the IOMMU.
1219 */
1220static void __domain_flush_pages(struct protection_domain *domain,
1221                                 u64 address, size_t size, int pde)
1222{
1223        struct iommu_dev_data *dev_data;
1224        struct iommu_cmd cmd;
1225        int ret = 0, i;
1226
1227        build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1228
1229        for (i = 0; i < amd_iommus_present; ++i) {
1230                if (!domain->dev_iommu[i])
1231                        continue;
1232
1233                /*
1234                 * Devices of this domain are behind this IOMMU
1235                 * We need a TLB flush
1236                 */
1237                ret |= iommu_queue_command(amd_iommus[i], &cmd);
1238        }
1239
1240        list_for_each_entry(dev_data, &domain->dev_list, list) {
1241
1242                if (!dev_data->ats.enabled)
1243                        continue;
1244
1245                ret |= device_flush_iotlb(dev_data, address, size);
1246        }
1247
1248        WARN_ON(ret);
1249}
1250
1251static void domain_flush_pages(struct protection_domain *domain,
1252                               u64 address, size_t size)
1253{
1254        __domain_flush_pages(domain, address, size, 0);
1255}
1256
1257/* Flush the whole IO/TLB for a given protection domain */
1258static void domain_flush_tlb(struct protection_domain *domain)
1259{
1260        __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1261}
1262
1263/* Flush the whole IO/TLB for a given protection domain - including PDE */
1264static void domain_flush_tlb_pde(struct protection_domain *domain)
1265{
1266        __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1267}
1268
1269static void domain_flush_complete(struct protection_domain *domain)
1270{
1271        int i;
1272
1273        for (i = 0; i < amd_iommus_present; ++i) {
1274                if (!domain->dev_iommu[i])
1275                        continue;
1276
1277                /*
1278                 * Devices of this domain are behind this IOMMU
1279                 * We need to wait for completion of all commands.
1280                 */
1281                iommu_completion_wait(amd_iommus[i]);
1282        }
1283}
1284
1285
1286/*
1287 * This function flushes the DTEs for all devices in domain
1288 */
1289static void domain_flush_devices(struct protection_domain *domain)
1290{
1291        struct iommu_dev_data *dev_data;
1292
1293        list_for_each_entry(dev_data, &domain->dev_list, list)
1294                device_flush_dte(dev_data);
1295}
1296
1297/****************************************************************************
1298 *
1299 * The functions below are used the create the page table mappings for
1300 * unity mapped regions.
1301 *
1302 ****************************************************************************/
1303
1304/*
1305 * This function is used to add another level to an IO page table. Adding
1306 * another level increases the size of the address space by 9 bits to a size up
1307 * to 64 bits.
1308 */
1309static bool increase_address_space(struct protection_domain *domain,
1310                                   gfp_t gfp)
1311{
1312        u64 *pte;
1313
1314        if (domain->mode == PAGE_MODE_6_LEVEL)
1315                /* address space already 64 bit large */
1316                return false;
1317
1318        pte = (void *)get_zeroed_page(gfp);
1319        if (!pte)
1320                return false;
1321
1322        *pte             = PM_LEVEL_PDE(domain->mode,
1323                                        virt_to_phys(domain->pt_root));
1324        domain->pt_root  = pte;
1325        domain->mode    += 1;
1326        domain->updated  = true;
1327
1328        return true;
1329}
1330
1331static u64 *alloc_pte(struct protection_domain *domain,
1332                      unsigned long address,
1333                      unsigned long page_size,
1334                      u64 **pte_page,
1335                      gfp_t gfp)
1336{
1337        int level, end_lvl;
1338        u64 *pte, *page;
1339
1340        BUG_ON(!is_power_of_2(page_size));
1341
1342        while (address > PM_LEVEL_SIZE(domain->mode))
1343                increase_address_space(domain, gfp);
1344
1345        level   = domain->mode - 1;
1346        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1347        address = PAGE_SIZE_ALIGN(address, page_size);
1348        end_lvl = PAGE_SIZE_LEVEL(page_size);
1349
1350        while (level > end_lvl) {
1351                if (!IOMMU_PTE_PRESENT(*pte)) {
1352                        page = (u64 *)get_zeroed_page(gfp);
1353                        if (!page)
1354                                return NULL;
1355                        *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
1356                }
1357
1358                /* No level skipping support yet */
1359                if (PM_PTE_LEVEL(*pte) != level)
1360                        return NULL;
1361
1362                level -= 1;
1363
1364                pte = IOMMU_PTE_PAGE(*pte);
1365
1366                if (pte_page && level == end_lvl)
1367                        *pte_page = pte;
1368
1369                pte = &pte[PM_LEVEL_INDEX(level, address)];
1370        }
1371
1372        return pte;
1373}
1374
1375/*
1376 * This function checks if there is a PTE for a given dma address. If
1377 * there is one, it returns the pointer to it.
1378 */
1379static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
1380{
1381        int level;
1382        u64 *pte;
1383
1384        if (address > PM_LEVEL_SIZE(domain->mode))
1385                return NULL;
1386
1387        level   =  domain->mode - 1;
1388        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1389
1390        while (level > 0) {
1391
1392                /* Not Present */
1393                if (!IOMMU_PTE_PRESENT(*pte))
1394                        return NULL;
1395
1396                /* Large PTE */
1397                if (PM_PTE_LEVEL(*pte) == 0x07) {
1398                        unsigned long pte_mask, __pte;
1399
1400                        /*
1401                         * If we have a series of large PTEs, make
1402                         * sure to return a pointer to the first one.
1403                         */
1404                        pte_mask = PTE_PAGE_SIZE(*pte);
1405                        pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
1406                        __pte    = ((unsigned long)pte) & pte_mask;
1407
1408                        return (u64 *)__pte;
1409                }
1410
1411                /* No level skipping support yet */
1412                if (PM_PTE_LEVEL(*pte) != level)
1413                        return NULL;
1414
1415                level -= 1;
1416
1417                /* Walk to the next level */
1418                pte = IOMMU_PTE_PAGE(*pte);
1419                pte = &pte[PM_LEVEL_INDEX(level, address)];
1420        }
1421
1422        return pte;
1423}
1424
1425/*
1426 * Generic mapping functions. It maps a physical address into a DMA
1427 * address space. It allocates the page table pages if necessary.
1428 * In the future it can be extended to a generic mapping function
1429 * supporting all features of AMD IOMMU page tables like level skipping
1430 * and full 64 bit address spaces.
1431 */
1432static int iommu_map_page(struct protection_domain *dom,
1433                          unsigned long bus_addr,
1434                          unsigned long phys_addr,
1435                          int prot,
1436                          unsigned long page_size)
1437{
1438        u64 __pte, *pte;
1439        int i, count;
1440
1441        if (!(prot & IOMMU_PROT_MASK))
1442                return -EINVAL;
1443
1444        bus_addr  = PAGE_ALIGN(bus_addr);
1445        phys_addr = PAGE_ALIGN(phys_addr);
1446        count     = PAGE_SIZE_PTE_COUNT(page_size);
1447        pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1448
1449        for (i = 0; i < count; ++i)
1450                if (IOMMU_PTE_PRESENT(pte[i]))
1451                        return -EBUSY;
1452
1453        if (page_size > PAGE_SIZE) {
1454                __pte = PAGE_SIZE_PTE(phys_addr, page_size);
1455                __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
1456        } else
1457                __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1458
1459        if (prot & IOMMU_PROT_IR)
1460                __pte |= IOMMU_PTE_IR;
1461        if (prot & IOMMU_PROT_IW)
1462                __pte |= IOMMU_PTE_IW;
1463
1464        for (i = 0; i < count; ++i)
1465                pte[i] = __pte;
1466
1467        update_domain(dom);
1468
1469        return 0;
1470}
1471
1472static unsigned long iommu_unmap_page(struct protection_domain *dom,
1473                                      unsigned long bus_addr,
1474                                      unsigned long page_size)
1475{
1476        unsigned long long unmap_size, unmapped;
1477        u64 *pte;
1478
1479        BUG_ON(!is_power_of_2(page_size));
1480
1481        unmapped = 0;
1482
1483        while (unmapped < page_size) {
1484
1485                pte = fetch_pte(dom, bus_addr);
1486
1487                if (!pte) {
1488                        /*
1489                         * No PTE for this address
1490                         * move forward in 4kb steps
1491                         */
1492                        unmap_size = PAGE_SIZE;
1493                } else if (PM_PTE_LEVEL(*pte) == 0) {
1494                        /* 4kb PTE found for this address */
1495                        unmap_size = PAGE_SIZE;
1496                        *pte       = 0ULL;
1497                } else {
1498                        int count, i;
1499
1500                        /* Large PTE found which maps this address */
1501                        unmap_size = PTE_PAGE_SIZE(*pte);
1502
1503                        /* Only unmap from the first pte in the page */
1504                        if ((unmap_size - 1) & bus_addr)
1505                                break;
1506                        count      = PAGE_SIZE_PTE_COUNT(unmap_size);
1507                        for (i = 0; i < count; i++)
1508                                pte[i] = 0ULL;
1509                }
1510
1511                bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
1512                unmapped += unmap_size;
1513        }
1514
1515        BUG_ON(unmapped && !is_power_of_2(unmapped));
1516
1517        return unmapped;
1518}
1519
1520/*
1521 * This function checks if a specific unity mapping entry is needed for
1522 * this specific IOMMU.
1523 */
1524static int iommu_for_unity_map(struct amd_iommu *iommu,
1525                               struct unity_map_entry *entry)
1526{
1527        u16 bdf, i;
1528
1529        for (i = entry->devid_start; i <= entry->devid_end; ++i) {
1530                bdf = amd_iommu_alias_table[i];
1531                if (amd_iommu_rlookup_table[bdf] == iommu)
1532                        return 1;
1533        }
1534
1535        return 0;
1536}
1537
1538/*
1539 * This function actually applies the mapping to the page table of the
1540 * dma_ops domain.
1541 */
1542static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
1543                             struct unity_map_entry *e)
1544{
1545        u64 addr;
1546        int ret;
1547
1548        for (addr = e->address_start; addr < e->address_end;
1549             addr += PAGE_SIZE) {
1550                ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1551                                     PAGE_SIZE);
1552                if (ret)
1553                        return ret;
1554                /*
1555                 * if unity mapping is in aperture range mark the page
1556                 * as allocated in the aperture
1557                 */
1558                if (addr < dma_dom->aperture_size)
1559                        __set_bit(addr >> PAGE_SHIFT,
1560                                  dma_dom->aperture[0]->bitmap);
1561        }
1562
1563        return 0;
1564}
1565
1566/*
1567 * Init the unity mappings for a specific IOMMU in the system
1568 *
1569 * Basically iterates over all unity mapping entries and applies them to
1570 * the default domain DMA of that IOMMU if necessary.
1571 */
1572static int iommu_init_unity_mappings(struct amd_iommu *iommu)
1573{
1574        struct unity_map_entry *entry;
1575        int ret;
1576
1577        list_for_each_entry(entry, &amd_iommu_unity_map, list) {
1578                if (!iommu_for_unity_map(iommu, entry))
1579                        continue;
1580                ret = dma_ops_unity_map(iommu->default_dom, entry);
1581                if (ret)
1582                        return ret;
1583        }
1584
1585        return 0;
1586}
1587
1588/*
1589 * Inits the unity mappings required for a specific device
1590 */
1591static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1592                                          u16 devid)
1593{
1594        struct unity_map_entry *e;
1595        int ret;
1596
1597        list_for_each_entry(e, &amd_iommu_unity_map, list) {
1598                if (!(devid >= e->devid_start && devid <= e->devid_end))
1599                        continue;
1600                ret = dma_ops_unity_map(dma_dom, e);
1601                if (ret)
1602                        return ret;
1603        }
1604
1605        return 0;
1606}
1607
1608/****************************************************************************
1609 *
1610 * The next functions belong to the address allocator for the dma_ops
1611 * interface functions. They work like the allocators in the other IOMMU
1612 * drivers. Its basically a bitmap which marks the allocated pages in
1613 * the aperture. Maybe it could be enhanced in the future to a more
1614 * efficient allocator.
1615 *
1616 ****************************************************************************/
1617
1618/*
1619 * The address allocator core functions.
1620 *
1621 * called with domain->lock held
1622 */
1623
1624/*
1625 * Used to reserve address ranges in the aperture (e.g. for exclusion
1626 * ranges.
1627 */
1628static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1629                                      unsigned long start_page,
1630                                      unsigned int pages)
1631{
1632        unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1633
1634        if (start_page + pages > last_page)
1635                pages = last_page - start_page;
1636
1637        for (i = start_page; i < start_page + pages; ++i) {
1638                int index = i / APERTURE_RANGE_PAGES;
1639                int page  = i % APERTURE_RANGE_PAGES;
1640                __set_bit(page, dom->aperture[index]->bitmap);
1641        }
1642}
1643
1644/*
1645 * This function is used to add a new aperture range to an existing
1646 * aperture in case of dma_ops domain allocation or address allocation
1647 * failure.
1648 */
1649static int alloc_new_range(struct dma_ops_domain *dma_dom,
1650                           bool populate, gfp_t gfp)
1651{
1652        int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1653        struct amd_iommu *iommu;
1654        unsigned long i, old_size;
1655
1656#ifdef CONFIG_IOMMU_STRESS
1657        populate = false;
1658#endif
1659
1660        if (index >= APERTURE_MAX_RANGES)
1661                return -ENOMEM;
1662
1663        dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
1664        if (!dma_dom->aperture[index])
1665                return -ENOMEM;
1666
1667        dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
1668        if (!dma_dom->aperture[index]->bitmap)
1669                goto out_free;
1670
1671        dma_dom->aperture[index]->offset = dma_dom->aperture_size;
1672
1673        if (populate) {
1674                unsigned long address = dma_dom->aperture_size;
1675                int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1676                u64 *pte, *pte_page;
1677
1678                for (i = 0; i < num_ptes; ++i) {
1679                        pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1680                                        &pte_page, gfp);
1681                        if (!pte)
1682                                goto out_free;
1683
1684                        dma_dom->aperture[index]->pte_pages[i] = pte_page;
1685
1686                        address += APERTURE_RANGE_SIZE / 64;
1687                }
1688        }
1689
1690        old_size                = dma_dom->aperture_size;
1691        dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1692
1693        /* Reserve address range used for MSI messages */
1694        if (old_size < MSI_ADDR_BASE_LO &&
1695            dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
1696                unsigned long spage;
1697                int pages;
1698
1699                pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
1700                spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
1701
1702                dma_ops_reserve_addresses(dma_dom, spage, pages);
1703        }
1704
1705        /* Initialize the exclusion range if necessary */
1706        for_each_iommu(iommu) {
1707                if (iommu->exclusion_start &&
1708                    iommu->exclusion_start >= dma_dom->aperture[index]->offset
1709                    && iommu->exclusion_start < dma_dom->aperture_size) {
1710                        unsigned long startpage;
1711                        int pages = iommu_num_pages(iommu->exclusion_start,
1712                                                    iommu->exclusion_length,
1713                                                    PAGE_SIZE);
1714                        startpage = iommu->exclusion_start >> PAGE_SHIFT;
1715                        dma_ops_reserve_addresses(dma_dom, startpage, pages);
1716                }
1717        }
1718
1719        /*
1720         * Check for areas already mapped as present in the new aperture
1721         * range and mark those pages as reserved in the allocator. Such
1722         * mappings may already exist as a result of requested unity
1723         * mappings for devices.
1724         */
1725        for (i = dma_dom->aperture[index]->offset;
1726             i < dma_dom->aperture_size;
1727             i += PAGE_SIZE) {
1728                u64 *pte = fetch_pte(&dma_dom->domain, i);
1729                if (!pte || !IOMMU_PTE_PRESENT(*pte))
1730                        continue;
1731
1732                dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
1733        }
1734
1735        update_domain(&dma_dom->domain);
1736
1737        return 0;
1738
1739out_free:
1740        update_domain(&dma_dom->domain);
1741
1742        free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1743
1744        kfree(dma_dom->aperture[index]);
1745        dma_dom->aperture[index] = NULL;
1746
1747        return -ENOMEM;
1748}
1749
1750static unsigned long dma_ops_area_alloc(struct device *dev,
1751                                        struct dma_ops_domain *dom,
1752                                        unsigned int pages,
1753                                        unsigned long align_mask,
1754                                        u64 dma_mask,
1755                                        unsigned long start)
1756{
1757        unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1758        int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1759        int i = start >> APERTURE_RANGE_SHIFT;
1760        unsigned long boundary_size;
1761        unsigned long address = -1;
1762        unsigned long limit;
1763
1764        next_bit >>= PAGE_SHIFT;
1765
1766        boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1767                        PAGE_SIZE) >> PAGE_SHIFT;
1768
1769        for (;i < max_index; ++i) {
1770                unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1771
1772                if (dom->aperture[i]->offset >= dma_mask)
1773                        break;
1774
1775                limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1776                                               dma_mask >> PAGE_SHIFT);
1777
1778                address = iommu_area_alloc(dom->aperture[i]->bitmap,
1779                                           limit, next_bit, pages, 0,
1780                                            boundary_size, align_mask);
1781                if (address != -1) {
1782                        address = dom->aperture[i]->offset +
1783                                  (address << PAGE_SHIFT);
1784                        dom->next_address = address + (pages << PAGE_SHIFT);
1785                        break;
1786                }
1787
1788                next_bit = 0;
1789        }
1790
1791        return address;
1792}
1793
1794static unsigned long dma_ops_alloc_addresses(struct device *dev,
1795                                             struct dma_ops_domain *dom,
1796                                             unsigned int pages,
1797                                             unsigned long align_mask,
1798                                             u64 dma_mask)
1799{
1800        unsigned long address;
1801
1802#ifdef CONFIG_IOMMU_STRESS
1803        dom->next_address = 0;
1804        dom->need_flush = true;
1805#endif
1806
1807        address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1808                                     dma_mask, dom->next_address);
1809
1810        if (address == -1) {
1811                dom->next_address = 0;
1812                address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1813                                             dma_mask, 0);
1814                dom->need_flush = true;
1815        }
1816
1817        if (unlikely(address == -1))
1818                address = DMA_ERROR_CODE;
1819
1820        WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1821
1822        return address;
1823}
1824
1825/*
1826 * The address free function.
1827 *
1828 * called with domain->lock held
1829 */
1830static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1831                                   unsigned long address,
1832                                   unsigned int pages)
1833{
1834        unsigned i = address >> APERTURE_RANGE_SHIFT;
1835        struct aperture_range *range = dom->aperture[i];
1836
1837        BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1838
1839#ifdef CONFIG_IOMMU_STRESS
1840        if (i < 4)
1841                return;
1842#endif
1843
1844        if (address >= dom->next_address)
1845                dom->need_flush = true;
1846
1847        address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1848
1849        bitmap_clear(range->bitmap, address, pages);
1850
1851}
1852
1853/****************************************************************************
1854 *
1855 * The next functions belong to the domain allocation. A domain is
1856 * allocated for every IOMMU as the default domain. If device isolation
1857 * is enabled, every device get its own domain. The most important thing
1858 * about domains is the page table mapping the DMA address space they
1859 * contain.
1860 *
1861 ****************************************************************************/
1862
1863/*
1864 * This function adds a protection domain to the global protection domain list
1865 */
1866static void add_domain_to_list(struct protection_domain *domain)
1867{
1868        unsigned long flags;
1869
1870        spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1871        list_add(&domain->list, &amd_iommu_pd_list);
1872        spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1873}
1874
1875/*
1876 * This function removes a protection domain to the global
1877 * protection domain list
1878 */
1879static void del_domain_from_list(struct protection_domain *domain)
1880{
1881        unsigned long flags;
1882
1883        spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1884        list_del(&domain->list);
1885        spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1886}
1887
1888static u16 domain_id_alloc(void)
1889{
1890        unsigned long flags;
1891        int id;
1892
1893        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1894        id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1895        BUG_ON(id == 0);
1896        if (id > 0 && id < MAX_DOMAIN_ID)
1897                __set_bit(id, amd_iommu_pd_alloc_bitmap);
1898        else
1899                id = 0;
1900        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1901
1902        return id;
1903}
1904
1905static void domain_id_free(int id)
1906{
1907        unsigned long flags;
1908
1909        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1910        if (id > 0 && id < MAX_DOMAIN_ID)
1911                __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1912        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1913}
1914
1915#define DEFINE_FREE_PT_FN(LVL, FN)                              \
1916static void free_pt_##LVL (unsigned long __pt)                  \
1917{                                                               \
1918        unsigned long p;                                        \
1919        u64 *pt;                                                \
1920        int i;                                                  \
1921                                                                \
1922        pt = (u64 *)__pt;                                       \
1923                                                                \
1924        for (i = 0; i < 512; ++i) {                             \
1925                if (!IOMMU_PTE_PRESENT(pt[i]))                  \
1926                        continue;                               \
1927                                                                \
1928                p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);       \
1929                FN(p);                                          \
1930        }                                                       \
1931        free_page((unsigned long)pt);                           \
1932}
1933
1934DEFINE_FREE_PT_FN(l2, free_page)
1935DEFINE_FREE_PT_FN(l3, free_pt_l2)
1936DEFINE_FREE_PT_FN(l4, free_pt_l3)
1937DEFINE_FREE_PT_FN(l5, free_pt_l4)
1938DEFINE_FREE_PT_FN(l6, free_pt_l5)
1939
1940static void free_pagetable(struct protection_domain *domain)
1941{
1942        unsigned long root = (unsigned long)domain->pt_root;
1943
1944        switch (domain->mode) {
1945        case PAGE_MODE_NONE:
1946                break;
1947        case PAGE_MODE_1_LEVEL:
1948                free_page(root);
1949                break;
1950        case PAGE_MODE_2_LEVEL:
1951                free_pt_l2(root);
1952                break;
1953        case PAGE_MODE_3_LEVEL:
1954                free_pt_l3(root);
1955                break;
1956        case PAGE_MODE_4_LEVEL:
1957                free_pt_l4(root);
1958                break;
1959        case PAGE_MODE_5_LEVEL:
1960                free_pt_l5(root);
1961                break;
1962        case PAGE_MODE_6_LEVEL:
1963                free_pt_l6(root);
1964                break;
1965        default:
1966                BUG();
1967        }
1968}
1969
1970static void free_gcr3_tbl_level1(u64 *tbl)
1971{
1972        u64 *ptr;
1973        int i;
1974
1975        for (i = 0; i < 512; ++i) {
1976                if (!(tbl[i] & GCR3_VALID))
1977                        continue;
1978
1979                ptr = __va(tbl[i] & PAGE_MASK);
1980
1981                free_page((unsigned long)ptr);
1982        }
1983}
1984
1985static void free_gcr3_tbl_level2(u64 *tbl)
1986{
1987        u64 *ptr;
1988        int i;
1989
1990        for (i = 0; i < 512; ++i) {
1991                if (!(tbl[i] & GCR3_VALID))
1992                        continue;
1993
1994                ptr = __va(tbl[i] & PAGE_MASK);
1995
1996                free_gcr3_tbl_level1(ptr);
1997        }
1998}
1999
2000static void free_gcr3_table(struct protection_domain *domain)
2001{
2002        if (domain->glx == 2)
2003                free_gcr3_tbl_level2(domain->gcr3_tbl);
2004        else if (domain->glx == 1)
2005                free_gcr3_tbl_level1(domain->gcr3_tbl);
2006        else if (domain->glx != 0)
2007                BUG();
2008
2009        free_page((unsigned long)domain->gcr3_tbl);
2010}
2011
2012/*
2013 * Free a domain, only used if something went wrong in the
2014 * allocation path and we need to free an already allocated page table
2015 */
2016static void dma_ops_domain_free(struct dma_ops_domain *dom)
2017{
2018        int i;
2019
2020        if (!dom)
2021                return;
2022
2023        del_domain_from_list(&dom->domain);
2024
2025        free_pagetable(&dom->domain);
2026
2027        for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
2028                if (!dom->aperture[i])
2029                        continue;
2030                free_page((unsigned long)dom->aperture[i]->bitmap);
2031                kfree(dom->aperture[i]);
2032        }
2033
2034        kfree(dom);
2035}
2036
2037/*
2038 * Allocates a new protection domain usable for the dma_ops functions.
2039 * It also initializes the page table and the address allocator data
2040 * structures required for the dma_ops interface
2041 */
2042static struct dma_ops_domain *dma_ops_domain_alloc(void)
2043{
2044        struct dma_ops_domain *dma_dom;
2045
2046        dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
2047        if (!dma_dom)
2048                return NULL;
2049
2050        spin_lock_init(&dma_dom->domain.lock);
2051
2052        dma_dom->domain.id = domain_id_alloc();
2053        if (dma_dom->domain.id == 0)
2054                goto free_dma_dom;
2055        INIT_LIST_HEAD(&dma_dom->domain.dev_list);
2056        dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
2057        dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2058        dma_dom->domain.flags = PD_DMA_OPS_MASK;
2059        dma_dom->domain.priv = dma_dom;
2060        if (!dma_dom->domain.pt_root)
2061                goto free_dma_dom;
2062
2063        dma_dom->need_flush = false;
2064        dma_dom->target_dev = 0xffff;
2065
2066        add_domain_to_list(&dma_dom->domain);
2067
2068        if (alloc_new_range(dma_dom, true, GFP_KERNEL))
2069                goto free_dma_dom;
2070
2071        /*
2072         * mark the first page as allocated so we never return 0 as
2073         * a valid dma-address. So we can use 0 as error value
2074         */
2075        dma_dom->aperture[0]->bitmap[0] = 1;
2076        dma_dom->next_address = 0;
2077
2078
2079        return dma_dom;
2080
2081free_dma_dom:
2082        dma_ops_domain_free(dma_dom);
2083
2084        return NULL;
2085}
2086
2087/*
2088 * little helper function to check whether a given protection domain is a
2089 * dma_ops domain
2090 */
2091static bool dma_ops_domain(struct protection_domain *domain)
2092{
2093        return domain->flags & PD_DMA_OPS_MASK;
2094}
2095
2096static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
2097{
2098        u64 pte_root = 0;
2099        u64 flags = 0;
2100
2101        if (domain->mode != PAGE_MODE_NONE)
2102                pte_root = virt_to_phys(domain->pt_root);
2103
2104        pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
2105                    << DEV_ENTRY_MODE_SHIFT;
2106        pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
2107
2108        flags = amd_iommu_dev_table[devid].data[1];
2109
2110        if (ats)
2111                flags |= DTE_FLAG_IOTLB;
2112
2113        if (domain->flags & PD_IOMMUV2_MASK) {
2114                u64 gcr3 = __pa(domain->gcr3_tbl);
2115                u64 glx  = domain->glx;
2116                u64 tmp;
2117
2118                pte_root |= DTE_FLAG_GV;
2119                pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
2120
2121                /* First mask out possible old values for GCR3 table */
2122                tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
2123                flags    &= ~tmp;
2124
2125                tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
2126                flags    &= ~tmp;
2127
2128                /* Encode GCR3 table into DTE */
2129                tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
2130                pte_root |= tmp;
2131
2132                tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
2133                flags    |= tmp;
2134
2135                tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
2136                flags    |= tmp;
2137        }
2138
2139        flags &= ~(0xffffUL);
2140        flags |= domain->id;
2141
2142        amd_iommu_dev_table[devid].data[1]  = flags;
2143        amd_iommu_dev_table[devid].data[0]  = pte_root;
2144}
2145
2146static void clear_dte_entry(u16 devid)
2147{
2148        /* remove entry from the device table seen by the hardware */
2149        amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
2150        amd_iommu_dev_table[devid].data[1] = 0;
2151
2152        amd_iommu_apply_erratum_63(devid);
2153}
2154
2155static void do_attach(struct iommu_dev_data *dev_data,
2156                      struct protection_domain *domain)
2157{
2158        struct amd_iommu *iommu;
2159        bool ats;
2160
2161        iommu = amd_iommu_rlookup_table[dev_data->devid];
2162        ats   = dev_data->ats.enabled;
2163
2164        /* Update data structures */
2165        dev_data->domain = domain;
2166        list_add(&dev_data->list, &domain->dev_list);
2167        set_dte_entry(dev_data->devid, domain, ats);
2168
2169        /* Do reference counting */
2170        domain->dev_iommu[iommu->index] += 1;
2171        domain->dev_cnt                 += 1;
2172
2173        /* Flush the DTE entry */
2174        device_flush_dte(dev_data);
2175}
2176
2177static void do_detach(struct iommu_dev_data *dev_data)
2178{
2179        struct amd_iommu *iommu;
2180
2181        iommu = amd_iommu_rlookup_table[dev_data->devid];
2182
2183        /* decrease reference counters */
2184        dev_data->domain->dev_iommu[iommu->index] -= 1;
2185        dev_data->domain->dev_cnt                 -= 1;
2186
2187        /* Update data structures */
2188        dev_data->domain = NULL;
2189        list_del(&dev_data->list);
2190        clear_dte_entry(dev_data->devid);
2191
2192        /* Flush the DTE entry */
2193        device_flush_dte(dev_data);
2194}
2195
2196/*
2197 * If a device is not yet associated with a domain, this function does
2198 * assigns it visible for the hardware
2199 */
2200static int __attach_device(struct iommu_dev_data *dev_data,
2201                           struct protection_domain *domain)
2202{
2203        int ret;
2204
2205        /* lock domain */
2206        spin_lock(&domain->lock);
2207
2208        if (dev_data->alias_data != NULL) {
2209                struct iommu_dev_data *alias_data = dev_data->alias_data;
2210
2211                /* Some sanity checks */
2212                ret = -EBUSY;
2213                if (alias_data->domain != NULL &&
2214                                alias_data->domain != domain)
2215                        goto out_unlock;
2216
2217                if (dev_data->domain != NULL &&
2218                                dev_data->domain != domain)
2219                        goto out_unlock;
2220
2221                /* Do real assignment */
2222                if (alias_data->domain == NULL)
2223                        do_attach(alias_data, domain);
2224
2225                atomic_inc(&alias_data->bind);
2226        }
2227
2228        if (dev_data->domain == NULL)
2229                do_attach(dev_data, domain);
2230
2231        atomic_inc(&dev_data->bind);
2232
2233        ret = 0;
2234
2235out_unlock:
2236
2237        /* ready */
2238        spin_unlock(&domain->lock);
2239
2240        return ret;
2241}
2242
2243
2244static void pdev_iommuv2_disable(struct pci_dev *pdev)
2245{
2246        pci_disable_ats(pdev);
2247        pci_disable_pri(pdev);
2248        pci_disable_pasid(pdev);
2249}
2250
2251/* FIXME: Change generic reset-function to do the same */
2252static int pri_reset_while_enabled(struct pci_dev *pdev)
2253{
2254        u16 control;
2255        int pos;
2256
2257        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2258        if (!pos)
2259                return -EINVAL;
2260
2261        pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
2262        control |= PCI_PRI_CTRL_RESET;
2263        pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2264
2265        return 0;
2266}
2267
2268static int pdev_iommuv2_enable(struct pci_dev *pdev)
2269{
2270        bool reset_enable;
2271        int reqs, ret;
2272
2273        /* FIXME: Hardcode number of outstanding requests for now */
2274        reqs = 32;
2275        if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
2276                reqs = 1;
2277        reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2278
2279        /* Only allow access to user-accessible pages */
2280        ret = pci_enable_pasid(pdev, 0);
2281        if (ret)
2282                goto out_err;
2283
2284        /* First reset the PRI state of the device */
2285        ret = pci_reset_pri(pdev);
2286        if (ret)
2287                goto out_err;
2288
2289        /* Enable PRI */
2290        ret = pci_enable_pri(pdev, reqs);
2291        if (ret)
2292                goto out_err;
2293
2294        if (reset_enable) {
2295                ret = pri_reset_while_enabled(pdev);
2296                if (ret)
2297                        goto out_err;
2298        }
2299
2300        ret = pci_enable_ats(pdev, PAGE_SHIFT);
2301        if (ret)
2302                goto out_err;
2303
2304        return 0;
2305
2306out_err:
2307        pci_disable_pri(pdev);
2308        pci_disable_pasid(pdev);
2309
2310        return ret;
2311}
2312
2313/* FIXME: Move this to PCI code */
2314#define PCI_PRI_TLP_OFF         (1 << 15)
2315
2316static bool pci_pri_tlp_required(struct pci_dev *pdev)
2317{
2318        u16 status;
2319        int pos;
2320
2321        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2322        if (!pos)
2323                return false;
2324
2325        pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2326
2327        return (status & PCI_PRI_TLP_OFF) ? true : false;
2328}
2329
2330/*
2331 * If a device is not yet associated with a domain, this function
2332 * assigns it visible for the hardware
2333 */
2334static int attach_device(struct device *dev,
2335                         struct protection_domain *domain)
2336{
2337        struct pci_dev *pdev = to_pci_dev(dev);
2338        struct iommu_dev_data *dev_data;
2339        unsigned long flags;
2340        int ret;
2341
2342        dev_data = get_dev_data(dev);
2343
2344        if (domain->flags & PD_IOMMUV2_MASK) {
2345                if (!dev_data->iommu_v2 || !dev_data->passthrough)
2346                        return -EINVAL;
2347
2348                if (pdev_iommuv2_enable(pdev) != 0)
2349                        return -EINVAL;
2350
2351                dev_data->ats.enabled = true;
2352                dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
2353                dev_data->pri_tlp     = pci_pri_tlp_required(pdev);
2354        } else if (amd_iommu_iotlb_sup &&
2355                   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2356                dev_data->ats.enabled = true;
2357                dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
2358        }
2359
2360        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2361        ret = __attach_device(dev_data, domain);
2362        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2363
2364        /*
2365         * We might boot into a crash-kernel here. The crashed kernel
2366         * left the caches in the IOMMU dirty. So we have to flush
2367         * here to evict all dirty stuff.
2368         */
2369        domain_flush_tlb_pde(domain);
2370
2371        return ret;
2372}
2373
2374/*
2375 * Removes a device from a protection domain (unlocked)
2376 */
2377static void __detach_device(struct iommu_dev_data *dev_data)
2378{
2379        struct protection_domain *domain;
2380        unsigned long flags;
2381
2382        BUG_ON(!dev_data->domain);
2383
2384        domain = dev_data->domain;
2385
2386        spin_lock_irqsave(&domain->lock, flags);
2387
2388        if (dev_data->alias_data != NULL) {
2389                struct iommu_dev_data *alias_data = dev_data->alias_data;
2390
2391                if (atomic_dec_and_test(&alias_data->bind))
2392                        do_detach(alias_data);
2393        }
2394
2395        if (atomic_dec_and_test(&dev_data->bind))
2396                do_detach(dev_data);
2397
2398        spin_unlock_irqrestore(&domain->lock, flags);
2399
2400        /*
2401         * If we run in passthrough mode the device must be assigned to the
2402         * passthrough domain if it is detached from any other domain.
2403         * Make sure we can deassign from the pt_domain itself.
2404         */
2405        if (dev_data->passthrough &&
2406            (dev_data->domain == NULL && domain != pt_domain))
2407                __attach_device(dev_data, pt_domain);
2408}
2409
2410/*
2411 * Removes a device from a protection domain (with devtable_lock held)
2412 */
2413static void detach_device(struct device *dev)
2414{
2415        struct protection_domain *domain;
2416        struct iommu_dev_data *dev_data;
2417        unsigned long flags;
2418
2419        dev_data = get_dev_data(dev);
2420        domain   = dev_data->domain;
2421
2422        /* lock device table */
2423        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2424        __detach_device(dev_data);
2425        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2426
2427        if (domain->flags & PD_IOMMUV2_MASK)
2428                pdev_iommuv2_disable(to_pci_dev(dev));
2429        else if (dev_data->ats.enabled)
2430                pci_disable_ats(to_pci_dev(dev));
2431
2432        dev_data->ats.enabled = false;
2433}
2434
2435/*
2436 * Find out the protection domain structure for a given PCI device. This
2437 * will give us the pointer to the page table root for example.
2438 */
2439static struct protection_domain *domain_for_device(struct device *dev)
2440{
2441        struct iommu_dev_data *dev_data;
2442        struct protection_domain *dom = NULL;
2443        unsigned long flags;
2444
2445        dev_data   = get_dev_data(dev);
2446
2447        if (dev_data->domain)
2448                return dev_data->domain;
2449
2450        if (dev_data->alias_data != NULL) {
2451                struct iommu_dev_data *alias_data = dev_data->alias_data;
2452
2453                read_lock_irqsave(&amd_iommu_devtable_lock, flags);
2454                if (alias_data->domain != NULL) {
2455                        __attach_device(dev_data, alias_data->domain);
2456                        dom = alias_data->domain;
2457                }
2458                read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2459        }
2460
2461        return dom;
2462}
2463
2464static int device_change_notifier(struct notifier_block *nb,
2465                                  unsigned long action, void *data)
2466{
2467        struct dma_ops_domain *dma_domain;
2468        struct protection_domain *domain;
2469        struct iommu_dev_data *dev_data;
2470        struct device *dev = data;
2471        struct amd_iommu *iommu;
2472        unsigned long flags;
2473        u16 devid;
2474
2475        if (!check_device(dev))
2476                return 0;
2477
2478        devid    = get_device_id(dev);
2479        iommu    = amd_iommu_rlookup_table[devid];
2480        dev_data = get_dev_data(dev);
2481
2482        switch (action) {
2483        case BUS_NOTIFY_UNBOUND_DRIVER:
2484
2485                domain = domain_for_device(dev);
2486
2487                if (!domain)
2488                        goto out;
2489                if (dev_data->passthrough)
2490                        break;
2491                detach_device(dev);
2492                break;
2493        case BUS_NOTIFY_ADD_DEVICE:
2494
2495                iommu_init_device(dev);
2496
2497                /*
2498                 * dev_data is still NULL and
2499                 * got initialized in iommu_init_device
2500                 */
2501                dev_data = get_dev_data(dev);
2502
2503                if (iommu_pass_through || dev_data->iommu_v2) {
2504                        dev_data->passthrough = true;
2505                        attach_device(dev, pt_domain);
2506                        break;
2507                }
2508
2509                domain = domain_for_device(dev);
2510
2511                /* allocate a protection domain if a device is added */
2512                dma_domain = find_protection_domain(devid);
2513                if (!dma_domain) {
2514                        dma_domain = dma_ops_domain_alloc();
2515                        if (!dma_domain)
2516                                goto out;
2517                        dma_domain->target_dev = devid;
2518
2519                        spin_lock_irqsave(&iommu_pd_list_lock, flags);
2520                        list_add_tail(&dma_domain->list, &iommu_pd_list);
2521                        spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
2522                }
2523
2524                dev->archdata.dma_ops = &amd_iommu_dma_ops;
2525
2526                break;
2527        case BUS_NOTIFY_DEL_DEVICE:
2528
2529                iommu_uninit_device(dev);
2530
2531        default:
2532                goto out;
2533        }
2534
2535        iommu_completion_wait(iommu);
2536
2537out:
2538        return 0;
2539}
2540
2541static struct notifier_block device_nb = {
2542        .notifier_call = device_change_notifier,
2543};
2544
2545void amd_iommu_init_notifier(void)
2546{
2547        bus_register_notifier(&pci_bus_type, &device_nb);
2548}
2549
2550/*****************************************************************************
2551 *
2552 * The next functions belong to the dma_ops mapping/unmapping code.
2553 *
2554 *****************************************************************************/
2555
2556/*
2557 * In the dma_ops path we only have the struct device. This function
2558 * finds the corresponding IOMMU, the protection domain and the
2559 * requestor id for a given device.
2560 * If the device is not yet associated with a domain this is also done
2561 * in this function.
2562 */
2563static struct protection_domain *get_domain(struct device *dev)
2564{
2565        struct protection_domain *domain;
2566        struct dma_ops_domain *dma_dom;
2567        u16 devid = get_device_id(dev);
2568
2569        if (!check_device(dev))
2570                return ERR_PTR(-EINVAL);
2571
2572        domain = domain_for_device(dev);
2573        if (domain != NULL && !dma_ops_domain(domain))
2574                return ERR_PTR(-EBUSY);
2575
2576        if (domain != NULL)
2577                return domain;
2578
2579        /* Device not bound yet - bind it */
2580        dma_dom = find_protection_domain(devid);
2581        if (!dma_dom)
2582                dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
2583        attach_device(dev, &dma_dom->domain);
2584        DUMP_printk("Using protection domain %d for device %s\n",
2585                    dma_dom->domain.id, dev_name(dev));
2586
2587        return &dma_dom->domain;
2588}
2589
2590static void update_device_table(struct protection_domain *domain)
2591{
2592        struct iommu_dev_data *dev_data;
2593
2594        list_for_each_entry(dev_data, &domain->dev_list, list)
2595                set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2596}
2597
2598static void update_domain(struct protection_domain *domain)
2599{
2600        if (!domain->updated)
2601                return;
2602
2603        update_device_table(domain);
2604
2605        domain_flush_devices(domain);
2606        domain_flush_tlb_pde(domain);
2607
2608        domain->updated = false;
2609}
2610
2611/*
2612 * This function fetches the PTE for a given address in the aperture
2613 */
2614static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
2615                            unsigned long address)
2616{
2617        struct aperture_range *aperture;
2618        u64 *pte, *pte_page;
2619
2620        aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2621        if (!aperture)
2622                return NULL;
2623
2624        pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2625        if (!pte) {
2626                pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2627                                GFP_ATOMIC);
2628                aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
2629        } else
2630                pte += PM_LEVEL_INDEX(0, address);
2631
2632        update_domain(&dom->domain);
2633
2634        return pte;
2635}
2636
2637/*
2638 * This is the generic map function. It maps one 4kb page at paddr to
2639 * the given address in the DMA address space for the domain.
2640 */
2641static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2642                                     unsigned long address,
2643                                     phys_addr_t paddr,
2644                                     int direction)
2645{
2646        u64 *pte, __pte;
2647
2648        WARN_ON(address > dom->aperture_size);
2649
2650        paddr &= PAGE_MASK;
2651
2652        pte  = dma_ops_get_pte(dom, address);
2653        if (!pte)
2654                return DMA_ERROR_CODE;
2655
2656        __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
2657
2658        if (direction == DMA_TO_DEVICE)
2659                __pte |= IOMMU_PTE_IR;
2660        else if (direction == DMA_FROM_DEVICE)
2661                __pte |= IOMMU_PTE_IW;
2662        else if (direction == DMA_BIDIRECTIONAL)
2663                __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
2664
2665        WARN_ON(*pte);
2666
2667        *pte = __pte;
2668
2669        return (dma_addr_t)address;
2670}
2671
2672/*
2673 * The generic unmapping function for on page in the DMA address space.
2674 */
2675static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2676                                 unsigned long address)
2677{
2678        struct aperture_range *aperture;
2679        u64 *pte;
2680
2681        if (address >= dom->aperture_size)
2682                return;
2683
2684        aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2685        if (!aperture)
2686                return;
2687
2688        pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2689        if (!pte)
2690                return;
2691
2692        pte += PM_LEVEL_INDEX(0, address);
2693
2694        WARN_ON(!*pte);
2695
2696        *pte = 0ULL;
2697}
2698
2699/*
2700 * This function contains common code for mapping of a physically
2701 * contiguous memory region into DMA address space. It is used by all
2702 * mapping functions provided with this IOMMU driver.
2703 * Must be called with the domain lock held.
2704 */
2705static dma_addr_t __map_single(struct device *dev,
2706                               struct dma_ops_domain *dma_dom,
2707                               phys_addr_t paddr,
2708                               size_t size,
2709                               int dir,
2710                               bool align,
2711                               u64 dma_mask)
2712{
2713        dma_addr_t offset = paddr & ~PAGE_MASK;
2714        dma_addr_t address, start, ret;
2715        unsigned int pages;
2716        unsigned long align_mask = 0;
2717        int i;
2718
2719        pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2720        paddr &= PAGE_MASK;
2721
2722        INC_STATS_COUNTER(total_map_requests);
2723
2724        if (pages > 1)
2725                INC_STATS_COUNTER(cross_page);
2726
2727        if (align)
2728                align_mask = (1UL << get_order(size)) - 1;
2729
2730retry:
2731        address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
2732                                          dma_mask);
2733        if (unlikely(address == DMA_ERROR_CODE)) {
2734                /*
2735                 * setting next_address here will let the address
2736                 * allocator only scan the new allocated range in the
2737                 * first run. This is a small optimization.
2738                 */
2739                dma_dom->next_address = dma_dom->aperture_size;
2740
2741                if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2742                        goto out;
2743
2744                /*
2745                 * aperture was successfully enlarged by 128 MB, try
2746                 * allocation again
2747                 */
2748                goto retry;
2749        }
2750
2751        start = address;
2752        for (i = 0; i < pages; ++i) {
2753                ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2754                if (ret == DMA_ERROR_CODE)
2755                        goto out_unmap;
2756
2757                paddr += PAGE_SIZE;
2758                start += PAGE_SIZE;
2759        }
2760        address += offset;
2761
2762        ADD_STATS_COUNTER(alloced_io_mem, size);
2763
2764        if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2765                domain_flush_tlb(&dma_dom->domain);
2766                dma_dom->need_flush = false;
2767        } else if (unlikely(amd_iommu_np_cache))
2768                domain_flush_pages(&dma_dom->domain, address, size);
2769
2770out:
2771        return address;
2772
2773out_unmap:
2774
2775        for (--i; i >= 0; --i) {
2776                start -= PAGE_SIZE;
2777                dma_ops_domain_unmap(dma_dom, start);
2778        }
2779
2780        dma_ops_free_addresses(dma_dom, address, pages);
2781
2782        return DMA_ERROR_CODE;
2783}
2784
2785/*
2786 * Does the reverse of the __map_single function. Must be called with
2787 * the domain lock held too
2788 */
2789static void __unmap_single(struct dma_ops_domain *dma_dom,
2790                           dma_addr_t dma_addr,
2791                           size_t size,
2792                           int dir)
2793{
2794        dma_addr_t flush_addr;
2795        dma_addr_t i, start;
2796        unsigned int pages;
2797
2798        if ((dma_addr == DMA_ERROR_CODE) ||
2799            (dma_addr + size > dma_dom->aperture_size))
2800                return;
2801
2802        flush_addr = dma_addr;
2803        pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2804        dma_addr &= PAGE_MASK;
2805        start = dma_addr;
2806
2807        for (i = 0; i < pages; ++i) {
2808                dma_ops_domain_unmap(dma_dom, start);
2809                start += PAGE_SIZE;
2810        }
2811
2812        SUB_STATS_COUNTER(alloced_io_mem, size);
2813
2814        dma_ops_free_addresses(dma_dom, dma_addr, pages);
2815
2816        if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2817                domain_flush_pages(&dma_dom->domain, flush_addr, size);
2818                dma_dom->need_flush = false;
2819        }
2820}
2821
2822/*
2823 * The exported map_single function for dma_ops.
2824 */
2825static dma_addr_t map_page(struct device *dev, struct page *page,
2826                           unsigned long offset, size_t size,
2827                           enum dma_data_direction dir,
2828                           struct dma_attrs *attrs)
2829{
2830        unsigned long flags;
2831        struct protection_domain *domain;
2832        dma_addr_t addr;
2833        u64 dma_mask;
2834        phys_addr_t paddr = page_to_phys(page) + offset;
2835
2836        INC_STATS_COUNTER(cnt_map_single);
2837
2838        domain = get_domain(dev);
2839        if (PTR_ERR(domain) == -EINVAL)
2840                return (dma_addr_t)paddr;
2841        else if (IS_ERR(domain))
2842                return DMA_ERROR_CODE;
2843
2844        dma_mask = *dev->dma_mask;
2845
2846        spin_lock_irqsave(&domain->lock, flags);
2847
2848        addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2849                            dma_mask);
2850        if (addr == DMA_ERROR_CODE)
2851                goto out;
2852
2853        domain_flush_complete(domain);
2854
2855out:
2856        spin_unlock_irqrestore(&domain->lock, flags);
2857
2858        return addr;
2859}
2860
2861/*
2862 * The exported unmap_single function for dma_ops.
2863 */
2864static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2865                       enum dma_data_direction dir, struct dma_attrs *attrs)
2866{
2867        unsigned long flags;
2868        struct protection_domain *domain;
2869
2870        INC_STATS_COUNTER(cnt_unmap_single);
2871
2872        domain = get_domain(dev);
2873        if (IS_ERR(domain))
2874                return;
2875
2876        spin_lock_irqsave(&domain->lock, flags);
2877
2878        __unmap_single(domain->priv, dma_addr, size, dir);
2879
2880        domain_flush_complete(domain);
2881
2882        spin_unlock_irqrestore(&domain->lock, flags);
2883}
2884
2885/*
2886 * The exported map_sg function for dma_ops (handles scatter-gather
2887 * lists).
2888 */
2889static int map_sg(struct device *dev, struct scatterlist *sglist,
2890                  int nelems, enum dma_data_direction dir,
2891                  struct dma_attrs *attrs)
2892{
2893        unsigned long flags;
2894        struct protection_domain *domain;
2895        int i;
2896        struct scatterlist *s;
2897        phys_addr_t paddr;
2898        int mapped_elems = 0;
2899        u64 dma_mask;
2900
2901        INC_STATS_COUNTER(cnt_map_sg);
2902
2903        domain = get_domain(dev);
2904        if (IS_ERR(domain))
2905                return 0;
2906
2907        dma_mask = *dev->dma_mask;
2908
2909        spin_lock_irqsave(&domain->lock, flags);
2910
2911        for_each_sg(sglist, s, nelems, i) {
2912                paddr = sg_phys(s);
2913
2914                s->dma_address = __map_single(dev, domain->priv,
2915                                              paddr, s->length, dir, false,
2916                                              dma_mask);
2917
2918                if (s->dma_address) {
2919                        s->dma_length = s->length;
2920                        mapped_elems++;
2921                } else
2922                        goto unmap;
2923        }
2924
2925        domain_flush_complete(domain);
2926
2927out:
2928        spin_unlock_irqrestore(&domain->lock, flags);
2929
2930        return mapped_elems;
2931unmap:
2932        for_each_sg(sglist, s, mapped_elems, i) {
2933                if (s->dma_address)
2934                        __unmap_single(domain->priv, s->dma_address,
2935                                       s->dma_length, dir);
2936                s->dma_address = s->dma_length = 0;
2937        }
2938
2939        mapped_elems = 0;
2940
2941        goto out;
2942}
2943
2944/*
2945 * The exported map_sg function for dma_ops (handles scatter-gather
2946 * lists).
2947 */
2948static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2949                     int nelems, enum dma_data_direction dir,
2950                     struct dma_attrs *attrs)
2951{
2952        unsigned long flags;
2953        struct protection_domain *domain;
2954        struct scatterlist *s;
2955        int i;
2956
2957        INC_STATS_COUNTER(cnt_unmap_sg);
2958
2959        domain = get_domain(dev);
2960        if (IS_ERR(domain))
2961                return;
2962
2963        spin_lock_irqsave(&domain->lock, flags);
2964
2965        for_each_sg(sglist, s, nelems, i) {
2966                __unmap_single(domain->priv, s->dma_address,
2967                               s->dma_length, dir);
2968                s->dma_address = s->dma_length = 0;
2969        }
2970
2971        domain_flush_complete(domain);
2972
2973        spin_unlock_irqrestore(&domain->lock, flags);
2974}
2975
2976/*
2977 * The exported alloc_coherent function for dma_ops.
2978 */
2979static void *alloc_coherent(struct device *dev, size_t size,
2980                            dma_addr_t *dma_addr, gfp_t flag,
2981                            struct dma_attrs *attrs)
2982{
2983        unsigned long flags;
2984        void *virt_addr;
2985        struct protection_domain *domain;
2986        phys_addr_t paddr;
2987        u64 dma_mask = dev->coherent_dma_mask;
2988
2989        INC_STATS_COUNTER(cnt_alloc_coherent);
2990
2991        domain = get_domain(dev);
2992        if (PTR_ERR(domain) == -EINVAL) {
2993                virt_addr = (void *)__get_free_pages(flag, get_order(size));
2994                *dma_addr = __pa(virt_addr);
2995                return virt_addr;
2996        } else if (IS_ERR(domain))
2997                return NULL;
2998
2999        dma_mask  = dev->coherent_dma_mask;
3000        flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
3001        flag     |= __GFP_ZERO;
3002
3003        virt_addr = (void *)__get_free_pages(flag, get_order(size));
3004        if (!virt_addr)
3005                return NULL;
3006
3007        paddr = virt_to_phys(virt_addr);
3008
3009        if (!dma_mask)
3010                dma_mask = *dev->dma_mask;
3011
3012        spin_lock_irqsave(&domain->lock, flags);
3013
3014        *dma_addr = __map_single(dev, domain->priv, paddr,
3015                                 size, DMA_BIDIRECTIONAL, true, dma_mask);
3016
3017        if (*dma_addr == DMA_ERROR_CODE) {
3018                spin_unlock_irqrestore(&domain->lock, flags);
3019                goto out_free;
3020        }
3021
3022        domain_flush_complete(domain);
3023
3024        spin_unlock_irqrestore(&domain->lock, flags);
3025
3026        return virt_addr;
3027
3028out_free:
3029
3030        free_pages((unsigned long)virt_addr, get_order(size));
3031
3032        return NULL;
3033}
3034
3035/*
3036 * The exported free_coherent function for dma_ops.
3037 */
3038static void free_coherent(struct device *dev, size_t size,
3039                          void *virt_addr, dma_addr_t dma_addr,
3040                          struct dma_attrs *attrs)
3041{
3042        unsigned long flags;
3043        struct protection_domain *domain;
3044
3045        INC_STATS_COUNTER(cnt_free_coherent);
3046
3047        domain = get_domain(dev);
3048        if (IS_ERR(domain))
3049                goto free_mem;
3050
3051        spin_lock_irqsave(&domain->lock, flags);
3052
3053        __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
3054
3055        domain_flush_complete(domain);
3056
3057        spin_unlock_irqrestore(&domain->lock, flags);
3058
3059free_mem:
3060        free_pages((unsigned long)virt_addr, get_order(size));
3061}
3062
3063/*
3064 * This function is called by the DMA layer to find out if we can handle a
3065 * particular device. It is part of the dma_ops.
3066 */
3067static int amd_iommu_dma_supported(struct device *dev, u64 mask)
3068{
3069        return check_device(dev);
3070}
3071
3072/*
3073 * The function for pre-allocating protection domains.
3074 *
3075 * If the driver core informs the DMA layer if a driver grabs a device
3076 * we don't need to preallocate the protection domains anymore.
3077 * For now we have to.
3078 */
3079static void __init prealloc_protection_domains(void)
3080{
3081        struct iommu_dev_data *dev_data;
3082        struct dma_ops_domain *dma_dom;
3083        struct pci_dev *dev = NULL;
3084        u16 devid;
3085
3086        for_each_pci_dev(dev) {
3087
3088                /* Do we handle this device? */
3089                if (!check_device(&dev->dev))
3090                        continue;
3091
3092                dev_data = get_dev_data(&dev->dev);
3093                if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
3094                        /* Make sure passthrough domain is allocated */
3095                        alloc_passthrough_domain();
3096                        dev_data->passthrough = true;
3097                        attach_device(&dev->dev, pt_domain);
3098                        pr_info("AMD-Vi: Using passthrough domain for device %s\n",
3099                                dev_name(&dev->dev));
3100                }
3101
3102                /* Is there already any domain for it? */
3103                if (domain_for_device(&dev->dev))
3104                        continue;
3105
3106                devid = get_device_id(&dev->dev);
3107
3108                dma_dom = dma_ops_domain_alloc();
3109                if (!dma_dom)
3110                        continue;
3111                init_unity_mappings_for_device(dma_dom, devid);
3112                dma_dom->target_dev = devid;
3113
3114                attach_device(&dev->dev, &dma_dom->domain);
3115
3116                list_add_tail(&dma_dom->list, &iommu_pd_list);
3117        }
3118}
3119
3120static struct dma_map_ops amd_iommu_dma_ops = {
3121        .alloc = alloc_coherent,
3122        .free = free_coherent,
3123        .map_page = map_page,
3124        .unmap_page = unmap_page,
3125        .map_sg = map_sg,
3126        .unmap_sg = unmap_sg,
3127        .dma_supported = amd_iommu_dma_supported,
3128};
3129
3130static unsigned device_dma_ops_init(void)
3131{
3132        struct iommu_dev_data *dev_data;
3133        struct pci_dev *pdev = NULL;
3134        unsigned unhandled = 0;
3135
3136        for_each_pci_dev(pdev) {
3137                if (!check_device(&pdev->dev)) {
3138
3139                        iommu_ignore_device(&pdev->dev);
3140
3141                        unhandled += 1;
3142                        continue;
3143                }
3144
3145                dev_data = get_dev_data(&pdev->dev);
3146
3147                if (!dev_data->passthrough)
3148                        pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
3149                else
3150                        pdev->dev.archdata.dma_ops = &nommu_dma_ops;
3151        }
3152
3153        return unhandled;
3154}
3155
3156/*
3157 * The function which clues the AMD IOMMU driver into dma_ops.
3158 */
3159
3160void __init amd_iommu_init_api(void)
3161{
3162        bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
3163}
3164
3165int __init amd_iommu_init_dma_ops(void)
3166{
3167        struct amd_iommu *iommu;
3168        int ret, unhandled;
3169
3170        /*
3171         * first allocate a default protection domain for every IOMMU we
3172         * found in the system. Devices not assigned to any other
3173         * protection domain will be assigned to the default one.
3174         */
3175        for_each_iommu(iommu) {
3176                iommu->default_dom = dma_ops_domain_alloc();
3177                if (iommu->default_dom == NULL)
3178                        return -ENOMEM;
3179                iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
3180                ret = iommu_init_unity_mappings(iommu);
3181                if (ret)
3182                        goto free_domains;
3183        }
3184
3185        /*
3186         * Pre-allocate the protection domains for each device.
3187         */
3188        prealloc_protection_domains();
3189
3190        iommu_detected = 1;
3191        swiotlb = 0;
3192
3193        /* Make the driver finally visible to the drivers */
3194        unhandled = device_dma_ops_init();
3195        if (unhandled && max_pfn > MAX_DMA32_PFN) {
3196                /* There are unhandled devices - initialize swiotlb for them */
3197                swiotlb = 1;
3198        }
3199
3200        amd_iommu_stats_init();
3201
3202        if (amd_iommu_unmap_flush)
3203                pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
3204        else
3205                pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
3206
3207        return 0;
3208
3209free_domains:
3210
3211        for_each_iommu(iommu) {
3212                dma_ops_domain_free(iommu->default_dom);
3213        }
3214
3215        return ret;
3216}
3217
3218/*****************************************************************************
3219 *
3220 * The following functions belong to the exported interface of AMD IOMMU
3221 *
3222 * This interface allows access to lower level functions of the IOMMU
3223 * like protection domain handling and assignement of devices to domains
3224 * which is not possible with the dma_ops interface.
3225 *
3226 *****************************************************************************/
3227
3228static void cleanup_domain(struct protection_domain *domain)
3229{
3230        struct iommu_dev_data *dev_data, *next;
3231        unsigned long flags;
3232
3233        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3234
3235        list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
3236                __detach_device(dev_data);
3237                atomic_set(&dev_data->bind, 0);
3238        }
3239
3240        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
3241}
3242
3243static void protection_domain_free(struct protection_domain *domain)
3244{
3245        if (!domain)
3246                return;
3247
3248        del_domain_from_list(domain);
3249
3250        if (domain->id)
3251                domain_id_free(domain->id);
3252
3253        kfree(domain);
3254}
3255
3256static struct protection_domain *protection_domain_alloc(void)
3257{
3258        struct protection_domain *domain;
3259
3260        domain = kzalloc(sizeof(*domain), GFP_KERNEL);
3261        if (!domain)
3262                return NULL;
3263
3264        spin_lock_init(&domain->lock);
3265        mutex_init(&domain->api_lock);
3266        domain->id = domain_id_alloc();
3267        if (!domain->id)
3268                goto out_err;
3269        INIT_LIST_HEAD(&domain->dev_list);
3270
3271        add_domain_to_list(domain);
3272
3273        return domain;
3274
3275out_err:
3276        kfree(domain);
3277
3278        return NULL;
3279}
3280
3281static int __init alloc_passthrough_domain(void)
3282{
3283        if (pt_domain != NULL)
3284                return 0;
3285
3286        /* allocate passthrough domain */
3287        pt_domain = protection_domain_alloc();
3288        if (!pt_domain)
3289                return -ENOMEM;
3290
3291        pt_domain->mode = PAGE_MODE_NONE;
3292
3293        return 0;
3294}
3295static int amd_iommu_domain_init(struct iommu_domain *dom)
3296{
3297        struct protection_domain *domain;
3298
3299        domain = protection_domain_alloc();
3300        if (!domain)
3301                goto out_free;
3302
3303        domain->mode    = PAGE_MODE_3_LEVEL;
3304        domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
3305        if (!domain->pt_root)
3306                goto out_free;
3307
3308        domain->iommu_domain = dom;
3309
3310        dom->priv = domain;
3311
3312        dom->geometry.aperture_start = 0;
3313        dom->geometry.aperture_end   = ~0ULL;
3314        dom->geometry.force_aperture = true;
3315
3316        return 0;
3317
3318out_free:
3319        protection_domain_free(domain);
3320
3321        return -ENOMEM;
3322}
3323
3324static void amd_iommu_domain_destroy(struct iommu_domain *dom)
3325{
3326        struct protection_domain *domain = dom->priv;
3327
3328        if (!domain)
3329                return;
3330
3331        if (domain->dev_cnt > 0)
3332                cleanup_domain(domain);
3333
3334        BUG_ON(domain->dev_cnt != 0);
3335
3336        if (domain->mode != PAGE_MODE_NONE)
3337                free_pagetable(domain);
3338
3339        if (domain->flags & PD_IOMMUV2_MASK)
3340                free_gcr3_table(domain);
3341
3342        protection_domain_free(domain);
3343
3344        dom->priv = NULL;
3345}
3346
3347static void amd_iommu_detach_device(struct iommu_domain *dom,
3348                                    struct device *dev)
3349{
3350        struct iommu_dev_data *dev_data = dev->archdata.iommu;
3351        struct amd_iommu *iommu;
3352        u16 devid;
3353
3354        if (!check_device(dev))
3355                return;
3356
3357        devid = get_device_id(dev);
3358
3359        if (dev_data->domain != NULL)
3360                detach_device(dev);
3361
3362        iommu = amd_iommu_rlookup_table[devid];
3363        if (!iommu)
3364                return;
3365
3366        iommu_completion_wait(iommu);
3367}
3368
3369static int amd_iommu_attach_device(struct iommu_domain *dom,
3370                                   struct device *dev)
3371{
3372        struct protection_domain *domain = dom->priv;
3373        struct iommu_dev_data *dev_data;
3374        struct amd_iommu *iommu;
3375        int ret;
3376
3377        if (!check_device(dev))
3378                return -EINVAL;
3379
3380        dev_data = dev->archdata.iommu;
3381
3382        iommu = amd_iommu_rlookup_table[dev_data->devid];
3383        if (!iommu)
3384                return -EINVAL;
3385
3386        if (dev_data->domain)
3387                detach_device(dev);
3388
3389        ret = attach_device(dev, domain);
3390
3391        iommu_completion_wait(iommu);
3392
3393        return ret;
3394}
3395
3396static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3397                         phys_addr_t paddr, size_t page_size, int iommu_prot)
3398{
3399        struct protection_domain *domain = dom->priv;
3400        int prot = 0;
3401        int ret;
3402
3403        if (domain->mode == PAGE_MODE_NONE)
3404                return -EINVAL;
3405
3406        if (iommu_prot & IOMMU_READ)
3407                prot |= IOMMU_PROT_IR;
3408        if (iommu_prot & IOMMU_WRITE)
3409                prot |= IOMMU_PROT_IW;
3410
3411        mutex_lock(&domain->api_lock);
3412        ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3413        mutex_unlock(&domain->api_lock);
3414
3415        return ret;
3416}
3417
3418static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
3419                           size_t page_size)
3420{
3421        struct protection_domain *domain = dom->priv;
3422        size_t unmap_size;
3423
3424        if (domain->mode == PAGE_MODE_NONE)
3425                return -EINVAL;
3426
3427        mutex_lock(&domain->api_lock);
3428        unmap_size = iommu_unmap_page(domain, iova, page_size);
3429        mutex_unlock(&domain->api_lock);
3430
3431        domain_flush_tlb_pde(domain);
3432
3433        return unmap_size;
3434}
3435
3436static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3437                                          dma_addr_t iova)
3438{
3439        struct protection_domain *domain = dom->priv;
3440        unsigned long offset_mask;
3441        phys_addr_t paddr;
3442        u64 *pte, __pte;
3443
3444        if (domain->mode == PAGE_MODE_NONE)
3445                return iova;
3446
3447        pte = fetch_pte(domain, iova);
3448
3449        if (!pte || !IOMMU_PTE_PRESENT(*pte))
3450                return 0;
3451
3452        if (PM_PTE_LEVEL(*pte) == 0)
3453                offset_mask = PAGE_SIZE - 1;
3454        else
3455                offset_mask = PTE_PAGE_SIZE(*pte) - 1;
3456
3457        __pte = *pte & PM_ADDR_MASK;
3458        paddr = (__pte & ~offset_mask) | (iova & offset_mask);
3459
3460        return paddr;
3461}
3462
3463static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
3464                                    unsigned long cap)
3465{
3466        switch (cap) {
3467        case IOMMU_CAP_CACHE_COHERENCY:
3468                return 1;
3469        case IOMMU_CAP_INTR_REMAP:
3470                return irq_remapping_enabled;
3471        }
3472
3473        return 0;
3474}
3475
3476static struct iommu_ops amd_iommu_ops = {
3477        .domain_init = amd_iommu_domain_init,
3478        .domain_destroy = amd_iommu_domain_destroy,
3479        .attach_dev = amd_iommu_attach_device,
3480        .detach_dev = amd_iommu_detach_device,
3481        .map = amd_iommu_map,
3482        .unmap = amd_iommu_unmap,
3483        .iova_to_phys = amd_iommu_iova_to_phys,
3484        .domain_has_cap = amd_iommu_domain_has_cap,
3485        .pgsize_bitmap  = AMD_IOMMU_PGSIZES,
3486};
3487
3488/*****************************************************************************
3489 *
3490 * The next functions do a basic initialization of IOMMU for pass through
3491 * mode
3492 *
3493 * In passthrough mode the IOMMU is initialized and enabled but not used for
3494 * DMA-API translation.
3495 *
3496 *****************************************************************************/
3497
3498int __init amd_iommu_init_passthrough(void)
3499{
3500        struct iommu_dev_data *dev_data;
3501        struct pci_dev *dev = NULL;
3502        struct amd_iommu *iommu;
3503        u16 devid;
3504        int ret;
3505
3506        ret = alloc_passthrough_domain();
3507        if (ret)
3508                return ret;
3509
3510        for_each_pci_dev(dev) {
3511                if (!check_device(&dev->dev))
3512                        continue;
3513
3514                dev_data = get_dev_data(&dev->dev);
3515                dev_data->passthrough = true;
3516
3517                devid = get_device_id(&dev->dev);
3518
3519                iommu = amd_iommu_rlookup_table[devid];
3520                if (!iommu)
3521                        continue;
3522
3523                attach_device(&dev->dev, pt_domain);
3524        }
3525
3526        amd_iommu_stats_init();
3527
3528        pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
3529
3530        return 0;
3531}
3532
3533/* IOMMUv2 specific functions */
3534int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3535{
3536        return atomic_notifier_chain_register(&ppr_notifier, nb);
3537}
3538EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
3539
3540int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
3541{
3542        return atomic_notifier_chain_unregister(&ppr_notifier, nb);
3543}
3544EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3545
3546void amd_iommu_domain_direct_map(struct iommu_domain *dom)
3547{
3548        struct protection_domain *domain = dom->priv;
3549        unsigned long flags;
3550
3551        spin_lock_irqsave(&domain->lock, flags);
3552
3553        /* Update data structure */
3554        domain->mode    = PAGE_MODE_NONE;
3555        domain->updated = true;
3556
3557        /* Make changes visible to IOMMUs */
3558        update_domain(domain);
3559
3560        /* Page-table is not visible to IOMMU anymore, so free it */
3561        free_pagetable(domain);
3562
3563        spin_unlock_irqrestore(&domain->lock, flags);
3564}
3565EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3566
3567int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
3568{
3569        struct protection_domain *domain = dom->priv;
3570        unsigned long flags;
3571        int levels, ret;
3572
3573        if (pasids <= 0 || pasids > (PASID_MASK + 1))
3574                return -EINVAL;
3575
3576        /* Number of GCR3 table levels required */
3577        for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
3578                levels += 1;
3579
3580        if (levels > amd_iommu_max_glx_val)
3581                return -EINVAL;
3582
3583        spin_lock_irqsave(&domain->lock, flags);
3584
3585        /*
3586         * Save us all sanity checks whether devices already in the
3587         * domain support IOMMUv2. Just force that the domain has no
3588         * devices attached when it is switched into IOMMUv2 mode.
3589         */
3590        ret = -EBUSY;
3591        if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
3592                goto out;
3593
3594        ret = -ENOMEM;
3595        domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
3596        if (domain->gcr3_tbl == NULL)
3597                goto out;
3598
3599        domain->glx      = levels;
3600        domain->flags   |= PD_IOMMUV2_MASK;
3601        domain->updated  = true;
3602
3603        update_domain(domain);
3604
3605        ret = 0;
3606
3607out:
3608        spin_unlock_irqrestore(&domain->lock, flags);
3609
3610        return ret;
3611}
3612EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3613
3614static int __flush_pasid(struct protection_domain *domain, int pasid,
3615                         u64 address, bool size)
3616{
3617        struct iommu_dev_data *dev_data;
3618        struct iommu_cmd cmd;
3619        int i, ret;
3620
3621        if (!(domain->flags & PD_IOMMUV2_MASK))
3622                return -EINVAL;
3623
3624        build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
3625
3626        /*
3627         * IOMMU TLB needs to be flushed before Device TLB to
3628         * prevent device TLB refill from IOMMU TLB
3629         */
3630        for (i = 0; i < amd_iommus_present; ++i) {
3631                if (domain->dev_iommu[i] == 0)
3632                        continue;
3633
3634                ret = iommu_queue_command(amd_iommus[i], &cmd);
3635                if (ret != 0)
3636                        goto out;
3637        }
3638
3639        /* Wait until IOMMU TLB flushes are complete */
3640        domain_flush_complete(domain);
3641
3642        /* Now flush device TLBs */
3643        list_for_each_entry(dev_data, &domain->dev_list, list) {
3644                struct amd_iommu *iommu;
3645                int qdep;
3646
3647                BUG_ON(!dev_data->ats.enabled);
3648
3649                qdep  = dev_data->ats.qdep;
3650                iommu = amd_iommu_rlookup_table[dev_data->devid];
3651
3652                build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
3653                                      qdep, address, size);
3654
3655                ret = iommu_queue_command(iommu, &cmd);
3656                if (ret != 0)
3657                        goto out;
3658        }
3659
3660        /* Wait until all device TLBs are flushed */
3661        domain_flush_complete(domain);
3662
3663        ret = 0;
3664
3665out:
3666
3667        return ret;
3668}
3669
3670static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
3671                                  u64 address)
3672{
3673        INC_STATS_COUNTER(invalidate_iotlb);
3674
3675        return __flush_pasid(domain, pasid, address, false);
3676}
3677
3678int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
3679                         u64 address)
3680{
3681        struct protection_domain *domain = dom->priv;
3682        unsigned long flags;
3683        int ret;
3684
3685        spin_lock_irqsave(&domain->lock, flags);
3686        ret = __amd_iommu_flush_page(domain, pasid, address);
3687        spin_unlock_irqrestore(&domain->lock, flags);
3688
3689        return ret;
3690}
3691EXPORT_SYMBOL(amd_iommu_flush_page);
3692
3693static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
3694{
3695        INC_STATS_COUNTER(invalidate_iotlb_all);
3696
3697        return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
3698                             true);
3699}
3700
3701int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
3702{
3703        struct protection_domain *domain = dom->priv;
3704        unsigned long flags;
3705        int ret;
3706
3707        spin_lock_irqsave(&domain->lock, flags);
3708        ret = __amd_iommu_flush_tlb(domain, pasid);
3709        spin_unlock_irqrestore(&domain->lock, flags);
3710
3711        return ret;
3712}
3713EXPORT_SYMBOL(amd_iommu_flush_tlb);
3714
3715static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
3716{
3717        int index;
3718        u64 *pte;
3719
3720        while (true) {
3721
3722                index = (pasid >> (9 * level)) & 0x1ff;
3723                pte   = &root[index];
3724
3725                if (level == 0)
3726                        break;
3727
3728                if (!(*pte & GCR3_VALID)) {
3729                        if (!alloc)
3730                                return NULL;
3731
3732                        root = (void *)get_zeroed_page(GFP_ATOMIC);
3733                        if (root == NULL)
3734                                return NULL;
3735
3736                        *pte = __pa(root) | GCR3_VALID;
3737                }
3738
3739                root = __va(*pte & PAGE_MASK);
3740
3741                level -= 1;
3742        }
3743
3744        return pte;
3745}
3746
3747static int __set_gcr3(struct protection_domain *domain, int pasid,
3748                      unsigned long cr3)
3749{
3750        u64 *pte;
3751
3752        if (domain->mode != PAGE_MODE_NONE)
3753                return -EINVAL;
3754
3755        pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
3756        if (pte == NULL)
3757                return -ENOMEM;
3758
3759        *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
3760
3761        return __amd_iommu_flush_tlb(domain, pasid);
3762}
3763
3764static int __clear_gcr3(struct protection_domain *domain, int pasid)
3765{
3766        u64 *pte;
3767
3768        if (domain->mode != PAGE_MODE_NONE)
3769                return -EINVAL;
3770
3771        pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
3772        if (pte == NULL)
3773                return 0;
3774
3775        *pte = 0;
3776
3777        return __amd_iommu_flush_tlb(domain, pasid);
3778}
3779
3780int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
3781                              unsigned long cr3)
3782{
3783        struct protection_domain *domain = dom->priv;
3784        unsigned long flags;
3785        int ret;
3786
3787        spin_lock_irqsave(&domain->lock, flags);
3788        ret = __set_gcr3(domain, pasid, cr3);
3789        spin_unlock_irqrestore(&domain->lock, flags);
3790
3791        return ret;
3792}
3793EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
3794
3795int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
3796{
3797        struct protection_domain *domain = dom->priv;
3798        unsigned long flags;
3799        int ret;
3800
3801        spin_lock_irqsave(&domain->lock, flags);
3802        ret = __clear_gcr3(domain, pasid);
3803        spin_unlock_irqrestore(&domain->lock, flags);
3804
3805        return ret;
3806}
3807EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3808
3809int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
3810                           int status, int tag)
3811{
3812        struct iommu_dev_data *dev_data;
3813        struct amd_iommu *iommu;
3814        struct iommu_cmd cmd;
3815
3816        INC_STATS_COUNTER(complete_ppr);
3817
3818        dev_data = get_dev_data(&pdev->dev);
3819        iommu    = amd_iommu_rlookup_table[dev_data->devid];
3820
3821        build_complete_ppr(&cmd, dev_data->devid, pasid, status,
3822                           tag, dev_data->pri_tlp);
3823
3824        return iommu_queue_command(iommu, &cmd);
3825}
3826EXPORT_SYMBOL(amd_iommu_complete_ppr);
3827
3828struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
3829{
3830        struct protection_domain *domain;
3831
3832        domain = get_domain(&pdev->dev);
3833        if (IS_ERR(domain))
3834                return NULL;
3835
3836        /* Only return IOMMUv2 domains */
3837        if (!(domain->flags & PD_IOMMUV2_MASK))
3838                return NULL;
3839
3840        return domain->iommu_domain;
3841}
3842EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3843
3844void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
3845{
3846        struct iommu_dev_data *dev_data;
3847
3848        if (!amd_iommu_v2_supported())
3849                return;
3850
3851        dev_data = get_dev_data(&pdev->dev);
3852        dev_data->errata |= (1 << erratum);
3853}
3854EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3855
3856int amd_iommu_device_info(struct pci_dev *pdev,
3857                          struct amd_iommu_device_info *info)
3858{
3859        int max_pasids;
3860        int pos;
3861
3862        if (pdev == NULL || info == NULL)
3863                return -EINVAL;
3864
3865        if (!amd_iommu_v2_supported())
3866                return -EINVAL;
3867
3868        memset(info, 0, sizeof(*info));
3869
3870        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
3871        if (pos)
3872                info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
3873
3874        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
3875        if (pos)
3876                info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
3877
3878        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
3879        if (pos) {
3880                int features;
3881
3882                max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
3883                max_pasids = min(max_pasids, (1 << 20));
3884
3885                info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
3886                info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
3887
3888                features = pci_pasid_features(pdev);
3889                if (features & PCI_PASID_CAP_EXEC)
3890                        info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
3891                if (features & PCI_PASID_CAP_PRIV)
3892                        info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
3893        }
3894
3895        return 0;
3896}
3897EXPORT_SYMBOL(amd_iommu_device_info);
3898
3899#ifdef CONFIG_IRQ_REMAP
3900
3901/*****************************************************************************
3902 *
3903 * Interrupt Remapping Implementation
3904 *
3905 *****************************************************************************/
3906
3907union irte {
3908        u32 val;
3909        struct {
3910                u32 valid       : 1,
3911                    no_fault    : 1,
3912                    int_type    : 3,
3913                    rq_eoi      : 1,
3914                    dm          : 1,
3915                    rsvd_1      : 1,
3916                    destination : 8,
3917                    vector      : 8,
3918                    rsvd_2      : 8;
3919        } fields;
3920};
3921
3922#define DTE_IRQ_PHYS_ADDR_MASK  (((1ULL << 45)-1) << 6)
3923#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
3924#define DTE_IRQ_TABLE_LEN       (8ULL << 1)
3925#define DTE_IRQ_REMAP_ENABLE    1ULL
3926
3927static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
3928{
3929        u64 dte;
3930
3931        dte     = amd_iommu_dev_table[devid].data[2];
3932        dte     &= ~DTE_IRQ_PHYS_ADDR_MASK;
3933        dte     |= virt_to_phys(table->table);
3934        dte     |= DTE_IRQ_REMAP_INTCTL;
3935        dte     |= DTE_IRQ_TABLE_LEN;
3936        dte     |= DTE_IRQ_REMAP_ENABLE;
3937
3938        amd_iommu_dev_table[devid].data[2] = dte;
3939}
3940
3941#define IRTE_ALLOCATED (~1U)
3942
3943static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
3944{
3945        struct irq_remap_table *table = NULL;
3946        struct amd_iommu *iommu;
3947        unsigned long flags;
3948        u16 alias;
3949
3950        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3951
3952        iommu = amd_iommu_rlookup_table[devid];
3953        if (!iommu)
3954                goto out_unlock;
3955
3956        table = irq_lookup_table[devid];
3957        if (table)
3958                goto out;
3959
3960        alias = amd_iommu_alias_table[devid];
3961        table = irq_lookup_table[alias];
3962        if (table) {
3963                irq_lookup_table[devid] = table;
3964                set_dte_irq_entry(devid, table);
3965                iommu_flush_dte(iommu, devid);
3966                goto out;
3967        }
3968
3969        /* Nothing there yet, allocate new irq remapping table */
3970        table = kzalloc(sizeof(*table), GFP_ATOMIC);
3971        if (!table)
3972                goto out;
3973
3974        /* Initialize table spin-lock */
3975        spin_lock_init(&table->lock);
3976
3977        if (ioapic)
3978                /* Keep the first 32 indexes free for IOAPIC interrupts */
3979                table->min_index = 32;
3980
3981        table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);
3982        if (!table->table) {
3983                kfree(table);
3984                table = NULL;
3985                goto out;
3986        }
3987
3988        memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));
3989
3990        if (ioapic) {
3991                int i;
3992
3993                for (i = 0; i < 32; ++i)
3994                        table->table[i] = IRTE_ALLOCATED;
3995        }
3996
3997        irq_lookup_table[devid] = table;
3998        set_dte_irq_entry(devid, table);
3999        iommu_flush_dte(iommu, devid);
4000        if (devid != alias) {
4001                irq_lookup_table[alias] = table;
4002                set_dte_irq_entry(devid, table);
4003                iommu_flush_dte(iommu, alias);
4004        }
4005
4006out:
4007        iommu_completion_wait(iommu);
4008
4009out_unlock:
4010        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
4011
4012        return table;
4013}
4014
4015static int alloc_irq_index(struct irq_cfg *cfg, u16 devid, int count)
4016{
4017        struct irq_remap_table *table;
4018        unsigned long flags;
4019        int index, c;
4020
4021        table = get_irq_table(devid, false);
4022        if (!table)
4023                return -ENODEV;
4024
4025        spin_lock_irqsave(&table->lock, flags);
4026
4027        /* Scan table for free entries */
4028        for (c = 0, index = table->min_index;
4029             index < MAX_IRQS_PER_TABLE;
4030             ++index) {
4031                if (table->table[index] == 0)
4032                        c += 1;
4033                else
4034                        c = 0;
4035
4036                if (c == count) {
4037                        struct irq_2_irte *irte_info;
4038
4039                        for (; c != 0; --c)
4040                                table->table[index - c + 1] = IRTE_ALLOCATED;
4041
4042                        index -= count - 1;
4043
4044                        cfg->remapped         = 1;
4045                        irte_info             = &cfg->irq_2_irte;
4046                        irte_info->devid      = devid;
4047                        irte_info->index      = index;
4048
4049                        goto out;
4050                }
4051        }
4052
4053        index = -ENOSPC;
4054
4055out:
4056        spin_unlock_irqrestore(&table->lock, flags);
4057
4058        return index;
4059}
4060
4061static int get_irte(u16 devid, int index, union irte *irte)
4062{
4063        struct irq_remap_table *table;
4064        unsigned long flags;
4065
4066        table = get_irq_table(devid, false);
4067        if (!table)
4068                return -ENOMEM;
4069
4070        spin_lock_irqsave(&table->lock, flags);
4071        irte->val = table->table[index];
4072        spin_unlock_irqrestore(&table->lock, flags);
4073
4074        return 0;
4075}
4076
4077static int modify_irte(u16 devid, int index, union irte irte)
4078{
4079        struct irq_remap_table *table;
4080        struct amd_iommu *iommu;
4081        unsigned long flags;
4082
4083        iommu = amd_iommu_rlookup_table[devid];
4084        if (iommu == NULL)
4085                return -EINVAL;
4086
4087        table = get_irq_table(devid, false);
4088        if (!table)
4089                return -ENOMEM;
4090
4091        spin_lock_irqsave(&table->lock, flags);
4092        table->table[index] = irte.val;
4093        spin_unlock_irqrestore(&table->lock, flags);
4094
4095        iommu_flush_irt(iommu, devid);
4096        iommu_completion_wait(iommu);
4097
4098        return 0;
4099}
4100
4101static void free_irte(u16 devid, int index)
4102{
4103        struct irq_remap_table *table;
4104        struct amd_iommu *iommu;
4105        unsigned long flags;
4106
4107        iommu = amd_iommu_rlookup_table[devid];
4108        if (iommu == NULL)
4109                return;
4110
4111        table = get_irq_table(devid, false);
4112        if (!table)
4113                return;
4114
4115        spin_lock_irqsave(&table->lock, flags);
4116        table->table[index] = 0;
4117        spin_unlock_irqrestore(&table->lock, flags);
4118
4119        iommu_flush_irt(iommu, devid);
4120        iommu_completion_wait(iommu);
4121}
4122
4123static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
4124                              unsigned int destination, int vector,
4125                              struct io_apic_irq_attr *attr)
4126{
4127        struct irq_remap_table *table;
4128        struct irq_2_irte *irte_info;
4129        struct irq_cfg *cfg;
4130        union irte irte;
4131        int ioapic_id;
4132        int index;
4133        int devid;
4134        int ret;
4135
4136        cfg = irq_get_chip_data(irq);
4137        if (!cfg)
4138                return -EINVAL;
4139
4140        irte_info = &cfg->irq_2_irte;
4141        ioapic_id = mpc_ioapic_id(attr->ioapic);
4142        devid     = get_ioapic_devid(ioapic_id);
4143
4144        if (devid < 0)
4145                return devid;
4146
4147        table = get_irq_table(devid, true);
4148        if (table == NULL)
4149                return -ENOMEM;
4150
4151        index = attr->ioapic_pin;
4152
4153        /* Setup IRQ remapping info */
4154        cfg->remapped         = 1;
4155        irte_info->devid      = devid;
4156        irte_info->index      = index;
4157
4158        /* Setup IRTE for IOMMU */
4159        irte.val                = 0;
4160        irte.fields.vector      = vector;
4161        irte.fields.int_type    = apic->irq_delivery_mode;
4162        irte.fields.destination = destination;
4163        irte.fields.dm          = apic->irq_dest_mode;
4164        irte.fields.valid       = 1;
4165
4166        ret = modify_irte(devid, index, irte);
4167        if (ret)
4168                return ret;
4169
4170        /* Setup IOAPIC entry */
4171        memset(entry, 0, sizeof(*entry));
4172
4173        entry->vector        = index;
4174        entry->mask          = 0;
4175        entry->trigger       = attr->trigger;
4176        entry->polarity      = attr->polarity;
4177
4178        /*
4179         * Mask level triggered irqs.
4180         */
4181        if (attr->trigger)
4182                entry->mask = 1;
4183
4184        return 0;
4185}
4186
4187static int set_affinity(struct irq_data *data, const struct cpumask *mask,
4188                        bool force)
4189{
4190        struct irq_2_irte *irte_info;
4191        unsigned int dest, irq;
4192        struct irq_cfg *cfg;
4193        union irte irte;
4194        int err;
4195
4196        if (!config_enabled(CONFIG_SMP))
4197                return -1;
4198
4199        cfg       = data->chip_data;
4200        irq       = data->irq;
4201        irte_info = &cfg->irq_2_irte;
4202
4203        if (!cpumask_intersects(mask, cpu_online_mask))
4204                return -EINVAL;
4205
4206        if (get_irte(irte_info->devid, irte_info->index, &irte))
4207                return -EBUSY;
4208
4209        if (assign_irq_vector(irq, cfg, mask))
4210                return -EBUSY;
4211
4212        err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);
4213        if (err) {
4214                if (assign_irq_vector(irq, cfg, data->affinity))
4215                        pr_err("AMD-Vi: Failed to recover vector for irq %d\n", irq);
4216                return err;
4217        }
4218
4219        irte.fields.vector      = cfg->vector;
4220        irte.fields.destination = dest;
4221
4222        modify_irte(irte_info->devid, irte_info->index, irte);
4223
4224        if (cfg->move_in_progress)
4225                send_cleanup_vector(cfg);
4226
4227        cpumask_copy(data->affinity, mask);
4228
4229        return 0;
4230}
4231
4232static int free_irq(int irq)
4233{
4234        struct irq_2_irte *irte_info;
4235        struct irq_cfg *cfg;
4236
4237        cfg = irq_get_chip_data(irq);
4238        if (!cfg)
4239                return -EINVAL;
4240
4241        irte_info = &cfg->irq_2_irte;
4242
4243        free_irte(irte_info->devid, irte_info->index);
4244
4245        return 0;
4246}
4247
4248static void compose_msi_msg(struct pci_dev *pdev,
4249                            unsigned int irq, unsigned int dest,
4250                            struct msi_msg *msg, u8 hpet_id)
4251{
4252        struct irq_2_irte *irte_info;
4253        struct irq_cfg *cfg;
4254        union irte irte;
4255
4256        cfg = irq_get_chip_data(irq);
4257        if (!cfg)
4258                return;
4259
4260        irte_info = &cfg->irq_2_irte;
4261
4262        irte.val                = 0;
4263        irte.fields.vector      = cfg->vector;
4264        irte.fields.int_type    = apic->irq_delivery_mode;
4265        irte.fields.destination = dest;
4266        irte.fields.dm          = apic->irq_dest_mode;
4267        irte.fields.valid       = 1;
4268
4269        modify_irte(irte_info->devid, irte_info->index, irte);
4270
4271        msg->address_hi = MSI_ADDR_BASE_HI;
4272        msg->address_lo = MSI_ADDR_BASE_LO;
4273        msg->data       = irte_info->index;
4274}
4275
4276static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec)
4277{
4278        struct irq_cfg *cfg;
4279        int index;
4280        u16 devid;
4281
4282        if (!pdev)
4283                return -EINVAL;
4284
4285        cfg = irq_get_chip_data(irq);
4286        if (!cfg)
4287                return -EINVAL;
4288
4289        devid = get_device_id(&pdev->dev);
4290        index = alloc_irq_index(cfg, devid, nvec);
4291
4292        return index < 0 ? MAX_IRQS_PER_TABLE : index;
4293}
4294
4295static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
4296                         int index, int offset)
4297{
4298        struct irq_2_irte *irte_info;
4299        struct irq_cfg *cfg;
4300        u16 devid;
4301
4302        if (!pdev)
4303                return -EINVAL;
4304
4305        cfg = irq_get_chip_data(irq);
4306        if (!cfg)
4307                return -EINVAL;
4308
4309        if (index >= MAX_IRQS_PER_TABLE)
4310                return 0;
4311
4312        devid           = get_device_id(&pdev->dev);
4313        irte_info       = &cfg->irq_2_irte;
4314
4315        cfg->remapped         = 1;
4316        irte_info->devid      = devid;
4317        irte_info->index      = index + offset;
4318
4319        return 0;
4320}
4321
4322static int setup_hpet_msi(unsigned int irq, unsigned int id)
4323{
4324        struct irq_2_irte *irte_info;
4325        struct irq_cfg *cfg;
4326        int index, devid;
4327
4328        cfg = irq_get_chip_data(irq);
4329        if (!cfg)
4330                return -EINVAL;
4331
4332        irte_info = &cfg->irq_2_irte;
4333        devid     = get_hpet_devid(id);
4334        if (devid < 0)
4335                return devid;
4336
4337        index = alloc_irq_index(cfg, devid, 1);
4338        if (index < 0)
4339                return index;
4340
4341        cfg->remapped         = 1;
4342        irte_info->devid      = devid;
4343        irte_info->index      = index;
4344
4345        return 0;
4346}
4347
4348struct irq_remap_ops amd_iommu_irq_ops = {
4349        .supported              = amd_iommu_supported,
4350        .prepare                = amd_iommu_prepare,
4351        .enable                 = amd_iommu_enable,
4352        .disable                = amd_iommu_disable,
4353        .reenable               = amd_iommu_reenable,
4354        .enable_faulting        = amd_iommu_enable_faulting,
4355        .setup_ioapic_entry     = setup_ioapic_entry,
4356        .set_affinity           = set_affinity,
4357        .free_irq               = free_irq,
4358        .compose_msi_msg        = compose_msi_msg,
4359        .msi_alloc_irq          = msi_alloc_irq,
4360        .msi_setup_irq          = msi_setup_irq,
4361        .setup_hpet_msi         = setup_hpet_msi,
4362};
4363#endif
4364
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.