linux/drivers/iommu/amd_iommu.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
   3 * Author: Joerg Roedel <joerg.roedel@amd.com>
   4 *         Leo Duran <leo.duran@amd.com>
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License version 2 as published
   8 * by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope that it will be useful,
  11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 * GNU General Public License for more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program; if not, write to the Free Software
  17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18 */
  19
  20#include <linux/ratelimit.h>
  21#include <linux/pci.h>
  22#include <linux/pci-ats.h>
  23#include <linux/bitmap.h>
  24#include <linux/slab.h>
  25#include <linux/debugfs.h>
  26#include <linux/scatterlist.h>
  27#include <linux/dma-mapping.h>
  28#include <linux/iommu-helper.h>
  29#include <linux/iommu.h>
  30#include <linux/delay.h>
  31#include <linux/amd-iommu.h>
  32#include <linux/notifier.h>
  33#include <linux/export.h>
  34#include <asm/msidef.h>
  35#include <asm/proto.h>
  36#include <asm/iommu.h>
  37#include <asm/gart.h>
  38#include <asm/dma.h>
  39
  40#include "amd_iommu_proto.h"
  41#include "amd_iommu_types.h"
  42
  43#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
  44
  45#define LOOP_TIMEOUT    100000
  46
  47/*
  48 * This bitmap is used to advertise the page sizes our hardware support
  49 * to the IOMMU core, which will then use this information to split
  50 * physically contiguous memory regions it is mapping into page sizes
  51 * that we support.
  52 *
  53 * Traditionally the IOMMU core just handed us the mappings directly,
  54 * after making sure the size is an order of a 4KiB page and that the
  55 * mapping has natural alignment.
  56 *
  57 * To retain this behavior, we currently advertise that we support
  58 * all page sizes that are an order of 4KiB.
  59 *
  60 * If at some point we'd like to utilize the IOMMU core's new behavior,
  61 * we could change this to advertise the real page sizes we support.
  62 */
  63#define AMD_IOMMU_PGSIZES       (~0xFFFUL)
  64
  65static DEFINE_RWLOCK(amd_iommu_devtable_lock);
  66
  67/* A list of preallocated protection domains */
  68static LIST_HEAD(iommu_pd_list);
  69static DEFINE_SPINLOCK(iommu_pd_list_lock);
  70
  71/* List of all available dev_data structures */
  72static LIST_HEAD(dev_data_list);
  73static DEFINE_SPINLOCK(dev_data_list_lock);
  74
  75/*
  76 * Domain for untranslated devices - only allocated
  77 * if iommu=pt passed on kernel cmd line.
  78 */
  79static struct protection_domain *pt_domain;
  80
  81static struct iommu_ops amd_iommu_ops;
  82
  83static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
  84int amd_iommu_max_glx_val = -1;
  85
  86static struct dma_map_ops amd_iommu_dma_ops;
  87
  88/*
  89 * general struct to manage commands send to an IOMMU
  90 */
  91struct iommu_cmd {
  92        u32 data[4];
  93};
  94
  95static void update_domain(struct protection_domain *domain);
  96static int __init alloc_passthrough_domain(void);
  97
  98/****************************************************************************
  99 *
 100 * Helper functions
 101 *
 102 ****************************************************************************/
 103
 104static struct iommu_dev_data *alloc_dev_data(u16 devid)
 105{
 106        struct iommu_dev_data *dev_data;
 107        unsigned long flags;
 108
 109        dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
 110        if (!dev_data)
 111                return NULL;
 112
 113        dev_data->devid = devid;
 114        atomic_set(&dev_data->bind, 0);
 115
 116        spin_lock_irqsave(&dev_data_list_lock, flags);
 117        list_add_tail(&dev_data->dev_data_list, &dev_data_list);
 118        spin_unlock_irqrestore(&dev_data_list_lock, flags);
 119
 120        return dev_data;
 121}
 122
 123static void free_dev_data(struct iommu_dev_data *dev_data)
 124{
 125        unsigned long flags;
 126
 127        spin_lock_irqsave(&dev_data_list_lock, flags);
 128        list_del(&dev_data->dev_data_list);
 129        spin_unlock_irqrestore(&dev_data_list_lock, flags);
 130
 131        kfree(dev_data);
 132}
 133
 134static struct iommu_dev_data *search_dev_data(u16 devid)
 135{
 136        struct iommu_dev_data *dev_data;
 137        unsigned long flags;
 138
 139        spin_lock_irqsave(&dev_data_list_lock, flags);
 140        list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
 141                if (dev_data->devid == devid)
 142                        goto out_unlock;
 143        }
 144
 145        dev_data = NULL;
 146
 147out_unlock:
 148        spin_unlock_irqrestore(&dev_data_list_lock, flags);
 149
 150        return dev_data;
 151}
 152
 153static struct iommu_dev_data *find_dev_data(u16 devid)
 154{
 155        struct iommu_dev_data *dev_data;
 156
 157        dev_data = search_dev_data(devid);
 158
 159        if (dev_data == NULL)
 160                dev_data = alloc_dev_data(devid);
 161
 162        return dev_data;
 163}
 164
 165static inline u16 get_device_id(struct device *dev)
 166{
 167        struct pci_dev *pdev = to_pci_dev(dev);
 168
 169        return calc_devid(pdev->bus->number, pdev->devfn);
 170}
 171
 172static struct iommu_dev_data *get_dev_data(struct device *dev)
 173{
 174        return dev->archdata.iommu;
 175}
 176
 177static bool pci_iommuv2_capable(struct pci_dev *pdev)
 178{
 179        static const int caps[] = {
 180                PCI_EXT_CAP_ID_ATS,
 181                PCI_EXT_CAP_ID_PRI,
 182                PCI_EXT_CAP_ID_PASID,
 183        };
 184        int i, pos;
 185
 186        for (i = 0; i < 3; ++i) {
 187                pos = pci_find_ext_capability(pdev, caps[i]);
 188                if (pos == 0)
 189                        return false;
 190        }
 191
 192        return true;
 193}
 194
 195static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
 196{
 197        struct iommu_dev_data *dev_data;
 198
 199        dev_data = get_dev_data(&pdev->dev);
 200
 201        return dev_data->errata & (1 << erratum) ? true : false;
 202}
 203
 204/*
 205 * In this function the list of preallocated protection domains is traversed to
 206 * find the domain for a specific device
 207 */
 208static struct dma_ops_domain *find_protection_domain(u16 devid)
 209{
 210        struct dma_ops_domain *entry, *ret = NULL;
 211        unsigned long flags;
 212        u16 alias = amd_iommu_alias_table[devid];
 213
 214        if (list_empty(&iommu_pd_list))
 215                return NULL;
 216
 217        spin_lock_irqsave(&iommu_pd_list_lock, flags);
 218
 219        list_for_each_entry(entry, &iommu_pd_list, list) {
 220                if (entry->target_dev == devid ||
 221                    entry->target_dev == alias) {
 222                        ret = entry;
 223                        break;
 224                }
 225        }
 226
 227        spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
 228
 229        return ret;
 230}
 231
 232/*
 233 * This function checks if the driver got a valid device from the caller to
 234 * avoid dereferencing invalid pointers.
 235 */
 236static bool check_device(struct device *dev)
 237{
 238        u16 devid;
 239
 240        if (!dev || !dev->dma_mask)
 241                return false;
 242
 243        /* No device or no PCI device */
 244        if (dev->bus != &pci_bus_type)
 245                return false;
 246
 247        devid = get_device_id(dev);
 248
 249        /* Out of our scope? */
 250        if (devid > amd_iommu_last_bdf)
 251                return false;
 252
 253        if (amd_iommu_rlookup_table[devid] == NULL)
 254                return false;
 255
 256        return true;
 257}
 258
 259static void swap_pci_ref(struct pci_dev **from, struct pci_dev *to)
 260{
 261        pci_dev_put(*from);
 262        *from = to;
 263}
 264
 265#define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
 266
 267static int iommu_init_device(struct device *dev)
 268{
 269        struct pci_dev *dma_pdev = NULL, *pdev = to_pci_dev(dev);
 270        struct iommu_dev_data *dev_data;
 271        struct iommu_group *group;
 272        u16 alias;
 273        int ret;
 274
 275        if (dev->archdata.iommu)
 276                return 0;
 277
 278        dev_data = find_dev_data(get_device_id(dev));
 279        if (!dev_data)
 280                return -ENOMEM;
 281
 282        alias = amd_iommu_alias_table[dev_data->devid];
 283        if (alias != dev_data->devid) {
 284                struct iommu_dev_data *alias_data;
 285
 286                alias_data = find_dev_data(alias);
 287                if (alias_data == NULL) {
 288                        pr_err("AMD-Vi: Warning: Unhandled device %s\n",
 289                                        dev_name(dev));
 290                        free_dev_data(dev_data);
 291                        return -ENOTSUPP;
 292                }
 293                dev_data->alias_data = alias_data;
 294
 295                dma_pdev = pci_get_bus_and_slot(alias >> 8, alias & 0xff);
 296        }
 297
 298        if (dma_pdev == NULL)
 299                dma_pdev = pci_dev_get(pdev);
 300
 301        /* Account for quirked devices */
 302        swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
 303
 304        /*
 305         * If it's a multifunction device that does not support our
 306         * required ACS flags, add to the same group as function 0.
 307         */
 308        if (dma_pdev->multifunction &&
 309            !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS))
 310                swap_pci_ref(&dma_pdev,
 311                             pci_get_slot(dma_pdev->bus,
 312                                          PCI_DEVFN(PCI_SLOT(dma_pdev->devfn),
 313                                          0)));
 314
 315        /*
 316         * Devices on the root bus go through the iommu.  If that's not us,
 317         * find the next upstream device and test ACS up to the root bus.
 318         * Finding the next device may require skipping virtual buses.
 319         */
 320        while (!pci_is_root_bus(dma_pdev->bus)) {
 321                struct pci_bus *bus = dma_pdev->bus;
 322
 323                while (!bus->self) {
 324                        if (!pci_is_root_bus(bus))
 325                                bus = bus->parent;
 326                        else
 327                                goto root_bus;
 328                }
 329
 330                if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
 331                        break;
 332
 333                swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
 334        }
 335
 336root_bus:
 337        group = iommu_group_get(&dma_pdev->dev);
 338        pci_dev_put(dma_pdev);
 339        if (!group) {
 340                group = iommu_group_alloc();
 341                if (IS_ERR(group))
 342                        return PTR_ERR(group);
 343        }
 344
 345        ret = iommu_group_add_device(group, dev);
 346
 347        iommu_group_put(group);
 348
 349        if (ret)
 350                return ret;
 351
 352        if (pci_iommuv2_capable(pdev)) {
 353                struct amd_iommu *iommu;
 354
 355                iommu              = amd_iommu_rlookup_table[dev_data->devid];
 356                dev_data->iommu_v2 = iommu->is_iommu_v2;
 357        }
 358
 359        dev->archdata.iommu = dev_data;
 360
 361        return 0;
 362}
 363
 364static void iommu_ignore_device(struct device *dev)
 365{
 366        u16 devid, alias;
 367
 368        devid = get_device_id(dev);
 369        alias = amd_iommu_alias_table[devid];
 370
 371        memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
 372        memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
 373
 374        amd_iommu_rlookup_table[devid] = NULL;
 375        amd_iommu_rlookup_table[alias] = NULL;
 376}
 377
 378static void iommu_uninit_device(struct device *dev)
 379{
 380        iommu_group_remove_device(dev);
 381
 382        /*
 383         * Nothing to do here - we keep dev_data around for unplugged devices
 384         * and reuse it when the device is re-plugged - not doing so would
 385         * introduce a ton of races.
 386         */
 387}
 388
 389void __init amd_iommu_uninit_devices(void)
 390{
 391        struct iommu_dev_data *dev_data, *n;
 392        struct pci_dev *pdev = NULL;
 393
 394        for_each_pci_dev(pdev) {
 395
 396                if (!check_device(&pdev->dev))
 397                        continue;
 398
 399                iommu_uninit_device(&pdev->dev);
 400        }
 401
 402        /* Free all of our dev_data structures */
 403        list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
 404                free_dev_data(dev_data);
 405}
 406
 407int __init amd_iommu_init_devices(void)
 408{
 409        struct pci_dev *pdev = NULL;
 410        int ret = 0;
 411
 412        for_each_pci_dev(pdev) {
 413
 414                if (!check_device(&pdev->dev))
 415                        continue;
 416
 417                ret = iommu_init_device(&pdev->dev);
 418                if (ret == -ENOTSUPP)
 419                        iommu_ignore_device(&pdev->dev);
 420                else if (ret)
 421                        goto out_free;
 422        }
 423
 424        return 0;
 425
 426out_free:
 427
 428        amd_iommu_uninit_devices();
 429
 430        return ret;
 431}
 432#ifdef CONFIG_AMD_IOMMU_STATS
 433
 434/*
 435 * Initialization code for statistics collection
 436 */
 437
 438DECLARE_STATS_COUNTER(compl_wait);
 439DECLARE_STATS_COUNTER(cnt_map_single);
 440DECLARE_STATS_COUNTER(cnt_unmap_single);
 441DECLARE_STATS_COUNTER(cnt_map_sg);
 442DECLARE_STATS_COUNTER(cnt_unmap_sg);
 443DECLARE_STATS_COUNTER(cnt_alloc_coherent);
 444DECLARE_STATS_COUNTER(cnt_free_coherent);
 445DECLARE_STATS_COUNTER(cross_page);
 446DECLARE_STATS_COUNTER(domain_flush_single);
 447DECLARE_STATS_COUNTER(domain_flush_all);
 448DECLARE_STATS_COUNTER(alloced_io_mem);
 449DECLARE_STATS_COUNTER(total_map_requests);
 450DECLARE_STATS_COUNTER(complete_ppr);
 451DECLARE_STATS_COUNTER(invalidate_iotlb);
 452DECLARE_STATS_COUNTER(invalidate_iotlb_all);
 453DECLARE_STATS_COUNTER(pri_requests);
 454
 455static struct dentry *stats_dir;
 456static struct dentry *de_fflush;
 457
 458static void amd_iommu_stats_add(struct __iommu_counter *cnt)
 459{
 460        if (stats_dir == NULL)
 461                return;
 462
 463        cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
 464                                       &cnt->value);
 465}
 466
 467static void amd_iommu_stats_init(void)
 468{
 469        stats_dir = debugfs_create_dir("amd-iommu", NULL);
 470        if (stats_dir == NULL)
 471                return;
 472
 473        de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
 474                                         &amd_iommu_unmap_flush);
 475
 476        amd_iommu_stats_add(&compl_wait);
 477        amd_iommu_stats_add(&cnt_map_single);
 478        amd_iommu_stats_add(&cnt_unmap_single);
 479        amd_iommu_stats_add(&cnt_map_sg);
 480        amd_iommu_stats_add(&cnt_unmap_sg);
 481        amd_iommu_stats_add(&cnt_alloc_coherent);
 482        amd_iommu_stats_add(&cnt_free_coherent);
 483        amd_iommu_stats_add(&cross_page);
 484        amd_iommu_stats_add(&domain_flush_single);
 485        amd_iommu_stats_add(&domain_flush_all);
 486        amd_iommu_stats_add(&alloced_io_mem);
 487        amd_iommu_stats_add(&total_map_requests);
 488        amd_iommu_stats_add(&complete_ppr);
 489        amd_iommu_stats_add(&invalidate_iotlb);
 490        amd_iommu_stats_add(&invalidate_iotlb_all);
 491        amd_iommu_stats_add(&pri_requests);
 492}
 493
 494#endif
 495
 496/****************************************************************************
 497 *
 498 * Interrupt handling functions
 499 *
 500 ****************************************************************************/
 501
 502static void dump_dte_entry(u16 devid)
 503{
 504        int i;
 505
 506        for (i = 0; i < 4; ++i)
 507                pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
 508                        amd_iommu_dev_table[devid].data[i]);
 509}
 510
 511static void dump_command(unsigned long phys_addr)
 512{
 513        struct iommu_cmd *cmd = phys_to_virt(phys_addr);
 514        int i;
 515
 516        for (i = 0; i < 4; ++i)
 517                pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
 518}
 519
 520static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 521{
 522        int type, devid, domid, flags;
 523        volatile u32 *event = __evt;
 524        int count = 0;
 525        u64 address;
 526
 527retry:
 528        type    = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
 529        devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
 530        domid   = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
 531        flags   = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
 532        address = (u64)(((u64)event[3]) << 32) | event[2];
 533
 534        if (type == 0) {
 535                /* Did we hit the erratum? */
 536                if (++count == LOOP_TIMEOUT) {
 537                        pr_err("AMD-Vi: No event written to event log\n");
 538                        return;
 539                }
 540                udelay(1);
 541                goto retry;
 542        }
 543
 544        printk(KERN_ERR "AMD-Vi: Event logged [");
 545
 546        switch (type) {
 547        case EVENT_TYPE_ILL_DEV:
 548                printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
 549                       "address=0x%016llx flags=0x%04x]\n",
 550                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 551                       address, flags);
 552                dump_dte_entry(devid);
 553                break;
 554        case EVENT_TYPE_IO_FAULT:
 555                printk("IO_PAGE_FAULT device=%02x:%02x.%x "
 556                       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
 557                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 558                       domid, address, flags);
 559                break;
 560        case EVENT_TYPE_DEV_TAB_ERR:
 561                printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
 562                       "address=0x%016llx flags=0x%04x]\n",
 563                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 564                       address, flags);
 565                break;
 566        case EVENT_TYPE_PAGE_TAB_ERR:
 567                printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
 568                       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
 569                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 570                       domid, address, flags);
 571                break;
 572        case EVENT_TYPE_ILL_CMD:
 573                printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
 574                dump_command(address);
 575                break;
 576        case EVENT_TYPE_CMD_HARD_ERR:
 577                printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
 578                       "flags=0x%04x]\n", address, flags);
 579                break;
 580        case EVENT_TYPE_IOTLB_INV_TO:
 581                printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
 582                       "address=0x%016llx]\n",
 583                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 584                       address);
 585                break;
 586        case EVENT_TYPE_INV_DEV_REQ:
 587                printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
 588                       "address=0x%016llx flags=0x%04x]\n",
 589                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
 590                       address, flags);
 591                break;
 592        default:
 593                printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
 594        }
 595
 596        memset(__evt, 0, 4 * sizeof(u32));
 597}
 598
 599static void iommu_poll_events(struct amd_iommu *iommu)
 600{
 601        u32 head, tail;
 602        unsigned long flags;
 603
 604        spin_lock_irqsave(&iommu->lock, flags);
 605
 606        head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
 607        tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
 608
 609        while (head != tail) {
 610                iommu_print_event(iommu, iommu->evt_buf + head);
 611                head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
 612        }
 613
 614        writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
 615
 616        spin_unlock_irqrestore(&iommu->lock, flags);
 617}
 618
 619static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
 620{
 621        struct amd_iommu_fault fault;
 622
 623        INC_STATS_COUNTER(pri_requests);
 624
 625        if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
 626                pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
 627                return;
 628        }
 629
 630        fault.address   = raw[1];
 631        fault.pasid     = PPR_PASID(raw[0]);
 632        fault.device_id = PPR_DEVID(raw[0]);
 633        fault.tag       = PPR_TAG(raw[0]);
 634        fault.flags     = PPR_FLAGS(raw[0]);
 635
 636        atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
 637}
 638
 639static void iommu_poll_ppr_log(struct amd_iommu *iommu)
 640{
 641        unsigned long flags;
 642        u32 head, tail;
 643
 644        if (iommu->ppr_log == NULL)
 645                return;
 646
 647        /* enable ppr interrupts again */
 648        writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
 649
 650        spin_lock_irqsave(&iommu->lock, flags);
 651
 652        head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 653        tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
 654
 655        while (head != tail) {
 656                volatile u64 *raw;
 657                u64 entry[2];
 658                int i;
 659
 660                raw = (u64 *)(iommu->ppr_log + head);
 661
 662                /*
 663                 * Hardware bug: Interrupt may arrive before the entry is
 664                 * written to memory. If this happens we need to wait for the
 665                 * entry to arrive.
 666                 */
 667                for (i = 0; i < LOOP_TIMEOUT; ++i) {
 668                        if (PPR_REQ_TYPE(raw[0]) != 0)
 669                                break;
 670                        udelay(1);
 671                }
 672
 673                /* Avoid memcpy function-call overhead */
 674                entry[0] = raw[0];
 675                entry[1] = raw[1];
 676
 677                /*
 678                 * To detect the hardware bug we need to clear the entry
 679                 * back to zero.
 680                 */
 681                raw[0] = raw[1] = 0UL;
 682
 683                /* Update head pointer of hardware ring-buffer */
 684                head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
 685                writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 686
 687                /*
 688                 * Release iommu->lock because ppr-handling might need to
 689                 * re-aquire it
 690                 */
 691                spin_unlock_irqrestore(&iommu->lock, flags);
 692
 693                /* Handle PPR entry */
 694                iommu_handle_ppr_entry(iommu, entry);
 695
 696                spin_lock_irqsave(&iommu->lock, flags);
 697
 698                /* Refresh ring-buffer information */
 699                head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
 700                tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
 701        }
 702
 703        spin_unlock_irqrestore(&iommu->lock, flags);
 704}
 705
 706irqreturn_t amd_iommu_int_thread(int irq, void *data)
 707{
 708        struct amd_iommu *iommu;
 709
 710        for_each_iommu(iommu) {
 711                iommu_poll_events(iommu);
 712                iommu_poll_ppr_log(iommu);
 713        }
 714
 715        return IRQ_HANDLED;
 716}
 717
 718irqreturn_t amd_iommu_int_handler(int irq, void *data)
 719{
 720        return IRQ_WAKE_THREAD;
 721}
 722
 723/****************************************************************************
 724 *
 725 * IOMMU command queuing functions
 726 *
 727 ****************************************************************************/
 728
 729static int wait_on_sem(volatile u64 *sem)
 730{
 731        int i = 0;
 732
 733        while (*sem == 0 && i < LOOP_TIMEOUT) {
 734                udelay(1);
 735                i += 1;
 736        }
 737
 738        if (i == LOOP_TIMEOUT) {
 739                pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
 740                return -EIO;
 741        }
 742
 743        return 0;
 744}
 745
 746static void copy_cmd_to_buffer(struct amd_iommu *iommu,
 747                               struct iommu_cmd *cmd,
 748                               u32 tail)
 749{
 750        u8 *target;
 751
 752        target = iommu->cmd_buf + tail;
 753        tail   = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
 754
 755        /* Copy command to buffer */
 756        memcpy(target, cmd, sizeof(*cmd));
 757
 758        /* Tell the IOMMU about it */
 759        writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
 760}
 761
 762static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
 763{
 764        WARN_ON(address & 0x7ULL);
 765
 766        memset(cmd, 0, sizeof(*cmd));
 767        cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
 768        cmd->data[1] = upper_32_bits(__pa(address));
 769        cmd->data[2] = 1;
 770        CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
 771}
 772
 773static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
 774{
 775        memset(cmd, 0, sizeof(*cmd));
 776        cmd->data[0] = devid;
 777        CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
 778}
 779
 780static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
 781                                  size_t size, u16 domid, int pde)
 782{
 783        u64 pages;
 784        int s;
 785
 786        pages = iommu_num_pages(address, size, PAGE_SIZE);
 787        s     = 0;
 788
 789        if (pages > 1) {
 790                /*
 791                 * If we have to flush more than one page, flush all
 792                 * TLB entries for this domain
 793                 */
 794                address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
 795                s = 1;
 796        }
 797
 798        address &= PAGE_MASK;
 799
 800        memset(cmd, 0, sizeof(*cmd));
 801        cmd->data[1] |= domid;
 802        cmd->data[2]  = lower_32_bits(address);
 803        cmd->data[3]  = upper_32_bits(address);
 804        CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
 805        if (s) /* size bit - we flush more than one 4kb page */
 806                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 807        if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
 808                cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 809}
 810
 811static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
 812                                  u64 address, size_t size)
 813{
 814        u64 pages;
 815        int s;
 816
 817        pages = iommu_num_pages(address, size, PAGE_SIZE);
 818        s     = 0;
 819
 820        if (pages > 1) {
 821                /*
 822                 * If we have to flush more than one page, flush all
 823                 * TLB entries for this domain
 824                 */
 825                address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
 826                s = 1;
 827        }
 828
 829        address &= PAGE_MASK;
 830
 831        memset(cmd, 0, sizeof(*cmd));
 832        cmd->data[0]  = devid;
 833        cmd->data[0] |= (qdep & 0xff) << 24;
 834        cmd->data[1]  = devid;
 835        cmd->data[2]  = lower_32_bits(address);
 836        cmd->data[3]  = upper_32_bits(address);
 837        CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
 838        if (s)
 839                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 840}
 841
 842static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
 843                                  u64 address, bool size)
 844{
 845        memset(cmd, 0, sizeof(*cmd));
 846
 847        address &= ~(0xfffULL);
 848
 849        cmd->data[0]  = pasid & PASID_MASK;
 850        cmd->data[1]  = domid;
 851        cmd->data[2]  = lower_32_bits(address);
 852        cmd->data[3]  = upper_32_bits(address);
 853        cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 854        cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
 855        if (size)
 856                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 857        CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
 858}
 859
 860static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
 861                                  int qdep, u64 address, bool size)
 862{
 863        memset(cmd, 0, sizeof(*cmd));
 864
 865        address &= ~(0xfffULL);
 866
 867        cmd->data[0]  = devid;
 868        cmd->data[0] |= (pasid & 0xff) << 16;
 869        cmd->data[0] |= (qdep  & 0xff) << 24;
 870        cmd->data[1]  = devid;
 871        cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
 872        cmd->data[2]  = lower_32_bits(address);
 873        cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
 874        cmd->data[3]  = upper_32_bits(address);
 875        if (size)
 876                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 877        CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
 878}
 879
 880static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
 881                               int status, int tag, bool gn)
 882{
 883        memset(cmd, 0, sizeof(*cmd));
 884
 885        cmd->data[0]  = devid;
 886        if (gn) {
 887                cmd->data[1]  = pasid & PASID_MASK;
 888                cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
 889        }
 890        cmd->data[3]  = tag & 0x1ff;
 891        cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
 892
 893        CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
 894}
 895
 896static void build_inv_all(struct iommu_cmd *cmd)
 897{
 898        memset(cmd, 0, sizeof(*cmd));
 899        CMD_SET_TYPE(cmd, CMD_INV_ALL);
 900}
 901
 902/*
 903 * Writes the command to the IOMMUs command buffer and informs the
 904 * hardware about the new command.
 905 */
 906static int iommu_queue_command_sync(struct amd_iommu *iommu,
 907                                    struct iommu_cmd *cmd,
 908                                    bool sync)
 909{
 910        u32 left, tail, head, next_tail;
 911        unsigned long flags;
 912
 913        WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
 914
 915again:
 916        spin_lock_irqsave(&iommu->lock, flags);
 917
 918        head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
 919        tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
 920        next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
 921        left      = (head - next_tail) % iommu->cmd_buf_size;
 922
 923        if (left <= 2) {
 924                struct iommu_cmd sync_cmd;
 925                volatile u64 sem = 0;
 926                int ret;
 927
 928                build_completion_wait(&sync_cmd, (u64)&sem);
 929                copy_cmd_to_buffer(iommu, &sync_cmd, tail);
 930
 931                spin_unlock_irqrestore(&iommu->lock, flags);
 932
 933                if ((ret = wait_on_sem(&sem)) != 0)
 934                        return ret;
 935
 936                goto again;
 937        }
 938
 939        copy_cmd_to_buffer(iommu, cmd, tail);
 940
 941        /* We need to sync now to make sure all commands are processed */
 942        iommu->need_sync = sync;
 943
 944        spin_unlock_irqrestore(&iommu->lock, flags);
 945
 946        return 0;
 947}
 948
 949static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
 950{
 951        return iommu_queue_command_sync(iommu, cmd, true);
 952}
 953
 954/*
 955 * This function queues a completion wait command into the command
 956 * buffer of an IOMMU
 957 */
 958static int iommu_completion_wait(struct amd_iommu *iommu)
 959{
 960        struct iommu_cmd cmd;
 961        volatile u64 sem = 0;
 962        int ret;
 963
 964        if (!iommu->need_sync)
 965                return 0;
 966
 967        build_completion_wait(&cmd, (u64)&sem);
 968
 969        ret = iommu_queue_command_sync(iommu, &cmd, false);
 970        if (ret)
 971                return ret;
 972
 973        return wait_on_sem(&sem);
 974}
 975
 976static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
 977{
 978        struct iommu_cmd cmd;
 979
 980        build_inv_dte(&cmd, devid);
 981
 982        return iommu_queue_command(iommu, &cmd);
 983}
 984
 985static void iommu_flush_dte_all(struct amd_iommu *iommu)
 986{
 987        u32 devid;
 988
 989        for (devid = 0; devid <= 0xffff; ++devid)
 990                iommu_flush_dte(iommu, devid);
 991
 992        iommu_completion_wait(iommu);
 993}
 994
 995/*
 996 * This function uses heavy locking and may disable irqs for some time. But
 997 * this is no issue because it is only called during resume.
 998 */
 999static void iommu_flush_tlb_all(struct amd_iommu *iommu)
1000{
1001        u32 dom_id;
1002
1003        for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
1004                struct iommu_cmd cmd;
1005                build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1006                                      dom_id, 1);
1007                iommu_queue_command(iommu, &cmd);
1008        }
1009
1010        iommu_completion_wait(iommu);
1011}
1012
1013static void iommu_flush_all(struct amd_iommu *iommu)
1014{
1015        struct iommu_cmd cmd;
1016
1017        build_inv_all(&cmd);
1018
1019        iommu_queue_command(iommu, &cmd);
1020        iommu_completion_wait(iommu);
1021}
1022
1023void iommu_flush_all_caches(struct amd_iommu *iommu)
1024{
1025        if (iommu_feature(iommu, FEATURE_IA)) {
1026                iommu_flush_all(iommu);
1027        } else {
1028                iommu_flush_dte_all(iommu);
1029                iommu_flush_tlb_all(iommu);
1030        }
1031}
1032
1033/*
1034 * Command send function for flushing on-device TLB
1035 */
1036static int device_flush_iotlb(struct iommu_dev_data *dev_data,
1037                              u64 address, size_t size)
1038{
1039        struct amd_iommu *iommu;
1040        struct iommu_cmd cmd;
1041        int qdep;
1042
1043        qdep     = dev_data->ats.qdep;
1044        iommu    = amd_iommu_rlookup_table[dev_data->devid];
1045
1046        build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1047
1048        return iommu_queue_command(iommu, &cmd);
1049}
1050
1051/*
1052 * Command send function for invalidating a device table entry
1053 */
1054static int device_flush_dte(struct iommu_dev_data *dev_data)
1055{
1056        struct amd_iommu *iommu;
1057        int ret;
1058
1059        iommu = amd_iommu_rlookup_table[dev_data->devid];
1060
1061        ret = iommu_flush_dte(iommu, dev_data->devid);
1062        if (ret)
1063                return ret;
1064
1065        if (dev_data->ats.enabled)
1066                ret = device_flush_iotlb(dev_data, 0, ~0UL);
1067
1068        return ret;
1069}
1070
1071/*
1072 * TLB invalidation function which is called from the mapping functions.
1073 * It invalidates a single PTE if the range to flush is within a single
1074 * page. Otherwise it flushes the whole TLB of the IOMMU.
1075 */
1076static void __domain_flush_pages(struct protection_domain *domain,
1077                                 u64 address, size_t size, int pde)
1078{
1079        struct iommu_dev_data *dev_data;
1080        struct iommu_cmd cmd;
1081        int ret = 0, i;
1082
1083        build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1084
1085        for (i = 0; i < amd_iommus_present; ++i) {
1086                if (!domain->dev_iommu[i])
1087                        continue;
1088
1089                /*
1090                 * Devices of this domain are behind this IOMMU
1091                 * We need a TLB flush
1092                 */
1093                ret |= iommu_queue_command(amd_iommus[i], &cmd);
1094        }
1095
1096        list_for_each_entry(dev_data, &domain->dev_list, list) {
1097
1098                if (!dev_data->ats.enabled)
1099                        continue;
1100
1101                ret |= device_flush_iotlb(dev_data, address, size);
1102        }
1103
1104        WARN_ON(ret);
1105}
1106
1107static void domain_flush_pages(struct protection_domain *domain,
1108                               u64 address, size_t size)
1109{
1110        __domain_flush_pages(domain, address, size, 0);
1111}
1112
1113/* Flush the whole IO/TLB for a given protection domain */
1114static void domain_flush_tlb(struct protection_domain *domain)
1115{
1116        __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1117}
1118
1119/* Flush the whole IO/TLB for a given protection domain - including PDE */
1120static void domain_flush_tlb_pde(struct protection_domain *domain)
1121{
1122        __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1123}
1124
1125static void domain_flush_complete(struct protection_domain *domain)
1126{
1127        int i;
1128
1129        for (i = 0; i < amd_iommus_present; ++i) {
1130                if (!domain->dev_iommu[i])
1131                        continue;
1132
1133                /*
1134                 * Devices of this domain are behind this IOMMU
1135                 * We need to wait for completion of all commands.
1136                 */
1137                iommu_completion_wait(amd_iommus[i]);
1138        }
1139}
1140
1141
1142/*
1143 * This function flushes the DTEs for all devices in domain
1144 */
1145static void domain_flush_devices(struct protection_domain *domain)
1146{
1147        struct iommu_dev_data *dev_data;
1148
1149        list_for_each_entry(dev_data, &domain->dev_list, list)
1150                device_flush_dte(dev_data);
1151}
1152
1153/****************************************************************************
1154 *
1155 * The functions below are used the create the page table mappings for
1156 * unity mapped regions.
1157 *
1158 ****************************************************************************/
1159
1160/*
1161 * This function is used to add another level to an IO page table. Adding
1162 * another level increases the size of the address space by 9 bits to a size up
1163 * to 64 bits.
1164 */
1165static bool increase_address_space(struct protection_domain *domain,
1166                                   gfp_t gfp)
1167{
1168        u64 *pte;
1169
1170        if (domain->mode == PAGE_MODE_6_LEVEL)
1171                /* address space already 64 bit large */
1172                return false;
1173
1174        pte = (void *)get_zeroed_page(gfp);
1175        if (!pte)
1176                return false;
1177
1178        *pte             = PM_LEVEL_PDE(domain->mode,
1179                                        virt_to_phys(domain->pt_root));
1180        domain->pt_root  = pte;
1181        domain->mode    += 1;
1182        domain->updated  = true;
1183
1184        return true;
1185}
1186
1187static u64 *alloc_pte(struct protection_domain *domain,
1188                      unsigned long address,
1189                      unsigned long page_size,
1190                      u64 **pte_page,
1191                      gfp_t gfp)
1192{
1193        int level, end_lvl;
1194        u64 *pte, *page;
1195
1196        BUG_ON(!is_power_of_2(page_size));
1197
1198        while (address > PM_LEVEL_SIZE(domain->mode))
1199                increase_address_space(domain, gfp);
1200
1201        level   = domain->mode - 1;
1202        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1203        address = PAGE_SIZE_ALIGN(address, page_size);
1204        end_lvl = PAGE_SIZE_LEVEL(page_size);
1205
1206        while (level > end_lvl) {
1207                if (!IOMMU_PTE_PRESENT(*pte)) {
1208                        page = (u64 *)get_zeroed_page(gfp);
1209                        if (!page)
1210                                return NULL;
1211                        *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
1212                }
1213
1214                /* No level skipping support yet */
1215                if (PM_PTE_LEVEL(*pte) != level)
1216                        return NULL;
1217
1218                level -= 1;
1219
1220                pte = IOMMU_PTE_PAGE(*pte);
1221
1222                if (pte_page && level == end_lvl)
1223                        *pte_page = pte;
1224
1225                pte = &pte[PM_LEVEL_INDEX(level, address)];
1226        }
1227
1228        return pte;
1229}
1230
1231/*
1232 * This function checks if there is a PTE for a given dma address. If
1233 * there is one, it returns the pointer to it.
1234 */
1235static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
1236{
1237        int level;
1238        u64 *pte;
1239
1240        if (address > PM_LEVEL_SIZE(domain->mode))
1241                return NULL;
1242
1243        level   =  domain->mode - 1;
1244        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1245
1246        while (level > 0) {
1247
1248                /* Not Present */
1249                if (!IOMMU_PTE_PRESENT(*pte))
1250                        return NULL;
1251
1252                /* Large PTE */
1253                if (PM_PTE_LEVEL(*pte) == 0x07) {
1254                        unsigned long pte_mask, __pte;
1255
1256                        /*
1257                         * If we have a series of large PTEs, make
1258                         * sure to return a pointer to the first one.
1259                         */
1260                        pte_mask = PTE_PAGE_SIZE(*pte);
1261                        pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
1262                        __pte    = ((unsigned long)pte) & pte_mask;
1263
1264                        return (u64 *)__pte;
1265                }
1266
1267                /* No level skipping support yet */
1268                if (PM_PTE_LEVEL(*pte) != level)
1269                        return NULL;
1270
1271                level -= 1;
1272
1273                /* Walk to the next level */
1274                pte = IOMMU_PTE_PAGE(*pte);
1275                pte = &pte[PM_LEVEL_INDEX(level, address)];
1276        }
1277
1278        return pte;
1279}
1280
1281/*
1282 * Generic mapping functions. It maps a physical address into a DMA
1283 * address space. It allocates the page table pages if necessary.
1284 * In the future it can be extended to a generic mapping function
1285 * supporting all features of AMD IOMMU page tables like level skipping
1286 * and full 64 bit address spaces.
1287 */
1288static int iommu_map_page(struct protection_domain *dom,
1289                          unsigned long bus_addr,
1290                          unsigned long phys_addr,
1291                          int prot,
1292                          unsigned long page_size)
1293{
1294        u64 __pte, *pte;
1295        int i, count;
1296
1297        if (!(prot & IOMMU_PROT_MASK))
1298                return -EINVAL;
1299
1300        bus_addr  = PAGE_ALIGN(bus_addr);
1301        phys_addr = PAGE_ALIGN(phys_addr);
1302        count     = PAGE_SIZE_PTE_COUNT(page_size);
1303        pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1304
1305        for (i = 0; i < count; ++i)
1306                if (IOMMU_PTE_PRESENT(pte[i]))
1307                        return -EBUSY;
1308
1309        if (page_size > PAGE_SIZE) {
1310                __pte = PAGE_SIZE_PTE(phys_addr, page_size);
1311                __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
1312        } else
1313                __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1314
1315        if (prot & IOMMU_PROT_IR)
1316                __pte |= IOMMU_PTE_IR;
1317        if (prot & IOMMU_PROT_IW)
1318                __pte |= IOMMU_PTE_IW;
1319
1320        for (i = 0; i < count; ++i)
1321                pte[i] = __pte;
1322
1323        update_domain(dom);
1324
1325        return 0;
1326}
1327
1328static unsigned long iommu_unmap_page(struct protection_domain *dom,
1329                                      unsigned long bus_addr,
1330                                      unsigned long page_size)
1331{
1332        unsigned long long unmap_size, unmapped;
1333        u64 *pte;
1334
1335        BUG_ON(!is_power_of_2(page_size));
1336
1337        unmapped = 0;
1338
1339        while (unmapped < page_size) {
1340
1341                pte = fetch_pte(dom, bus_addr);
1342
1343                if (!pte) {
1344                        /*
1345                         * No PTE for this address
1346                         * move forward in 4kb steps
1347                         */
1348                        unmap_size = PAGE_SIZE;
1349                } else if (PM_PTE_LEVEL(*pte) == 0) {
1350                        /* 4kb PTE found for this address */
1351                        unmap_size = PAGE_SIZE;
1352                        *pte       = 0ULL;
1353                } else {
1354                        int count, i;
1355
1356                        /* Large PTE found which maps this address */
1357                        unmap_size = PTE_PAGE_SIZE(*pte);
1358                        count      = PAGE_SIZE_PTE_COUNT(unmap_size);
1359                        for (i = 0; i < count; i++)
1360                                pte[i] = 0ULL;
1361                }
1362
1363                bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
1364                unmapped += unmap_size;
1365        }
1366
1367        BUG_ON(!is_power_of_2(unmapped));
1368
1369        return unmapped;
1370}
1371
1372/*
1373 * This function checks if a specific unity mapping entry is needed for
1374 * this specific IOMMU.
1375 */
1376static int iommu_for_unity_map(struct amd_iommu *iommu,
1377                               struct unity_map_entry *entry)
1378{
1379        u16 bdf, i;
1380
1381        for (i = entry->devid_start; i <= entry->devid_end; ++i) {
1382                bdf = amd_iommu_alias_table[i];
1383                if (amd_iommu_rlookup_table[bdf] == iommu)
1384                        return 1;
1385        }
1386
1387        return 0;
1388}
1389
1390/*
1391 * This function actually applies the mapping to the page table of the
1392 * dma_ops domain.
1393 */
1394static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
1395                             struct unity_map_entry *e)
1396{
1397        u64 addr;
1398        int ret;
1399
1400        for (addr = e->address_start; addr < e->address_end;
1401             addr += PAGE_SIZE) {
1402                ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1403                                     PAGE_SIZE);
1404                if (ret)
1405                        return ret;
1406                /*
1407                 * if unity mapping is in aperture range mark the page
1408                 * as allocated in the aperture
1409                 */
1410                if (addr < dma_dom->aperture_size)
1411                        __set_bit(addr >> PAGE_SHIFT,
1412                                  dma_dom->aperture[0]->bitmap);
1413        }
1414
1415        return 0;
1416}
1417
1418/*
1419 * Init the unity mappings for a specific IOMMU in the system
1420 *
1421 * Basically iterates over all unity mapping entries and applies them to
1422 * the default domain DMA of that IOMMU if necessary.
1423 */
1424static int iommu_init_unity_mappings(struct amd_iommu *iommu)
1425{
1426        struct unity_map_entry *entry;
1427        int ret;
1428
1429        list_for_each_entry(entry, &amd_iommu_unity_map, list) {
1430                if (!iommu_for_unity_map(iommu, entry))
1431                        continue;
1432                ret = dma_ops_unity_map(iommu->default_dom, entry);
1433                if (ret)
1434                        return ret;
1435        }
1436
1437        return 0;
1438}
1439
1440/*
1441 * Inits the unity mappings required for a specific device
1442 */
1443static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1444                                          u16 devid)
1445{
1446        struct unity_map_entry *e;
1447        int ret;
1448
1449        list_for_each_entry(e, &amd_iommu_unity_map, list) {
1450                if (!(devid >= e->devid_start && devid <= e->devid_end))
1451                        continue;
1452                ret = dma_ops_unity_map(dma_dom, e);
1453                if (ret)
1454                        return ret;
1455        }
1456
1457        return 0;
1458}
1459
1460/****************************************************************************
1461 *
1462 * The next functions belong to the address allocator for the dma_ops
1463 * interface functions. They work like the allocators in the other IOMMU
1464 * drivers. Its basically a bitmap which marks the allocated pages in
1465 * the aperture. Maybe it could be enhanced in the future to a more
1466 * efficient allocator.
1467 *
1468 ****************************************************************************/
1469
1470/*
1471 * The address allocator core functions.
1472 *
1473 * called with domain->lock held
1474 */
1475
1476/*
1477 * Used to reserve address ranges in the aperture (e.g. for exclusion
1478 * ranges.
1479 */
1480static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1481                                      unsigned long start_page,
1482                                      unsigned int pages)
1483{
1484        unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1485
1486        if (start_page + pages > last_page)
1487                pages = last_page - start_page;
1488
1489        for (i = start_page; i < start_page + pages; ++i) {
1490                int index = i / APERTURE_RANGE_PAGES;
1491                int page  = i % APERTURE_RANGE_PAGES;
1492                __set_bit(page, dom->aperture[index]->bitmap);
1493        }
1494}
1495
1496/*
1497 * This function is used to add a new aperture range to an existing
1498 * aperture in case of dma_ops domain allocation or address allocation
1499 * failure.
1500 */
1501static int alloc_new_range(struct dma_ops_domain *dma_dom,
1502                           bool populate, gfp_t gfp)
1503{
1504        int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1505        struct amd_iommu *iommu;
1506        unsigned long i, old_size;
1507
1508#ifdef CONFIG_IOMMU_STRESS
1509        populate = false;
1510#endif
1511
1512        if (index >= APERTURE_MAX_RANGES)
1513                return -ENOMEM;
1514
1515        dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
1516        if (!dma_dom->aperture[index])
1517                return -ENOMEM;
1518
1519        dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
1520        if (!dma_dom->aperture[index]->bitmap)
1521                goto out_free;
1522
1523        dma_dom->aperture[index]->offset = dma_dom->aperture_size;
1524
1525        if (populate) {
1526                unsigned long address = dma_dom->aperture_size;
1527                int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1528                u64 *pte, *pte_page;
1529
1530                for (i = 0; i < num_ptes; ++i) {
1531                        pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1532                                        &pte_page, gfp);
1533                        if (!pte)
1534                                goto out_free;
1535
1536                        dma_dom->aperture[index]->pte_pages[i] = pte_page;
1537
1538                        address += APERTURE_RANGE_SIZE / 64;
1539                }
1540        }
1541
1542        old_size                = dma_dom->aperture_size;
1543        dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1544
1545        /* Reserve address range used for MSI messages */
1546        if (old_size < MSI_ADDR_BASE_LO &&
1547            dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
1548                unsigned long spage;
1549                int pages;
1550
1551                pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
1552                spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
1553
1554                dma_ops_reserve_addresses(dma_dom, spage, pages);
1555        }
1556
1557        /* Initialize the exclusion range if necessary */
1558        for_each_iommu(iommu) {
1559                if (iommu->exclusion_start &&
1560                    iommu->exclusion_start >= dma_dom->aperture[index]->offset
1561                    && iommu->exclusion_start < dma_dom->aperture_size) {
1562                        unsigned long startpage;
1563                        int pages = iommu_num_pages(iommu->exclusion_start,
1564                                                    iommu->exclusion_length,
1565                                                    PAGE_SIZE);
1566                        startpage = iommu->exclusion_start >> PAGE_SHIFT;
1567                        dma_ops_reserve_addresses(dma_dom, startpage, pages);
1568                }
1569        }
1570
1571        /*
1572         * Check for areas already mapped as present in the new aperture
1573         * range and mark those pages as reserved in the allocator. Such
1574         * mappings may already exist as a result of requested unity
1575         * mappings for devices.
1576         */
1577        for (i = dma_dom->aperture[index]->offset;
1578             i < dma_dom->aperture_size;
1579             i += PAGE_SIZE) {
1580                u64 *pte = fetch_pte(&dma_dom->domain, i);
1581                if (!pte || !IOMMU_PTE_PRESENT(*pte))
1582                        continue;
1583
1584                dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
1585        }
1586
1587        update_domain(&dma_dom->domain);
1588
1589        return 0;
1590
1591out_free:
1592        update_domain(&dma_dom->domain);
1593
1594        free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1595
1596        kfree(dma_dom->aperture[index]);
1597        dma_dom->aperture[index] = NULL;
1598
1599        return -ENOMEM;
1600}
1601
1602static unsigned long dma_ops_area_alloc(struct device *dev,
1603                                        struct dma_ops_domain *dom,
1604                                        unsigned int pages,
1605                                        unsigned long align_mask,
1606                                        u64 dma_mask,
1607                                        unsigned long start)
1608{
1609        unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1610        int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1611        int i = start >> APERTURE_RANGE_SHIFT;
1612        unsigned long boundary_size;
1613        unsigned long address = -1;
1614        unsigned long limit;
1615
1616        next_bit >>= PAGE_SHIFT;
1617
1618        boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1619                        PAGE_SIZE) >> PAGE_SHIFT;
1620
1621        for (;i < max_index; ++i) {
1622                unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1623
1624                if (dom->aperture[i]->offset >= dma_mask)
1625                        break;
1626
1627                limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1628                                               dma_mask >> PAGE_SHIFT);
1629
1630                address = iommu_area_alloc(dom->aperture[i]->bitmap,
1631                                           limit, next_bit, pages, 0,
1632                                            boundary_size, align_mask);
1633                if (address != -1) {
1634                        address = dom->aperture[i]->offset +
1635                                  (address << PAGE_SHIFT);
1636                        dom->next_address = address + (pages << PAGE_SHIFT);
1637                        break;
1638                }
1639
1640                next_bit = 0;
1641        }
1642
1643        return address;
1644}
1645
1646static unsigned long dma_ops_alloc_addresses(struct device *dev,
1647                                             struct dma_ops_domain *dom,
1648                                             unsigned int pages,
1649                                             unsigned long align_mask,
1650                                             u64 dma_mask)
1651{
1652        unsigned long address;
1653
1654#ifdef CONFIG_IOMMU_STRESS
1655        dom->next_address = 0;
1656        dom->need_flush = true;
1657#endif
1658
1659        address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1660                                     dma_mask, dom->next_address);
1661
1662        if (address == -1) {
1663                dom->next_address = 0;
1664                address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1665                                             dma_mask, 0);
1666                dom->need_flush = true;
1667        }
1668
1669        if (unlikely(address == -1))
1670                address = DMA_ERROR_CODE;
1671
1672        WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1673
1674        return address;
1675}
1676
1677/*
1678 * The address free function.
1679 *
1680 * called with domain->lock held
1681 */
1682static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1683                                   unsigned long address,
1684                                   unsigned int pages)
1685{
1686        unsigned i = address >> APERTURE_RANGE_SHIFT;
1687        struct aperture_range *range = dom->aperture[i];
1688
1689        BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1690
1691#ifdef CONFIG_IOMMU_STRESS
1692        if (i < 4)
1693                return;
1694#endif
1695
1696        if (address >= dom->next_address)
1697                dom->need_flush = true;
1698
1699        address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1700
1701        bitmap_clear(range->bitmap, address, pages);
1702
1703}
1704
1705/****************************************************************************
1706 *
1707 * The next functions belong to the domain allocation. A domain is
1708 * allocated for every IOMMU as the default domain. If device isolation
1709 * is enabled, every device get its own domain. The most important thing
1710 * about domains is the page table mapping the DMA address space they
1711 * contain.
1712 *
1713 ****************************************************************************/
1714
1715/*
1716 * This function adds a protection domain to the global protection domain list
1717 */
1718static void add_domain_to_list(struct protection_domain *domain)
1719{
1720        unsigned long flags;
1721
1722        spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1723        list_add(&domain->list, &amd_iommu_pd_list);
1724        spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1725}
1726
1727/*
1728 * This function removes a protection domain to the global
1729 * protection domain list
1730 */
1731static void del_domain_from_list(struct protection_domain *domain)
1732{
1733        unsigned long flags;
1734
1735        spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1736        list_del(&domain->list);
1737        spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1738}
1739
1740static u16 domain_id_alloc(void)
1741{
1742        unsigned long flags;
1743        int id;
1744
1745        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1746        id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1747        BUG_ON(id == 0);
1748        if (id > 0 && id < MAX_DOMAIN_ID)
1749                __set_bit(id, amd_iommu_pd_alloc_bitmap);
1750        else
1751                id = 0;
1752        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1753
1754        return id;
1755}
1756
1757static void domain_id_free(int id)
1758{
1759        unsigned long flags;
1760
1761        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1762        if (id > 0 && id < MAX_DOMAIN_ID)
1763                __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1764        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1765}
1766
1767static void free_pagetable(struct protection_domain *domain)
1768{
1769        int i, j;
1770        u64 *p1, *p2, *p3;
1771
1772        p1 = domain->pt_root;
1773
1774        if (!p1)
1775                return;
1776
1777        for (i = 0; i < 512; ++i) {
1778                if (!IOMMU_PTE_PRESENT(p1[i]))
1779                        continue;
1780
1781                p2 = IOMMU_PTE_PAGE(p1[i]);
1782                for (j = 0; j < 512; ++j) {
1783                        if (!IOMMU_PTE_PRESENT(p2[j]))
1784                                continue;
1785                        p3 = IOMMU_PTE_PAGE(p2[j]);
1786                        free_page((unsigned long)p3);
1787                }
1788
1789                free_page((unsigned long)p2);
1790        }
1791
1792        free_page((unsigned long)p1);
1793
1794        domain->pt_root = NULL;
1795}
1796
1797static void free_gcr3_tbl_level1(u64 *tbl)
1798{
1799        u64 *ptr;
1800        int i;
1801
1802        for (i = 0; i < 512; ++i) {
1803                if (!(tbl[i] & GCR3_VALID))
1804                        continue;
1805
1806                ptr = __va(tbl[i] & PAGE_MASK);
1807
1808                free_page((unsigned long)ptr);
1809        }
1810}
1811
1812static void free_gcr3_tbl_level2(u64 *tbl)
1813{
1814        u64 *ptr;
1815        int i;
1816
1817        for (i = 0; i < 512; ++i) {
1818                if (!(tbl[i] & GCR3_VALID))
1819                        continue;
1820
1821                ptr = __va(tbl[i] & PAGE_MASK);
1822
1823                free_gcr3_tbl_level1(ptr);
1824        }
1825}
1826
1827static void free_gcr3_table(struct protection_domain *domain)
1828{
1829        if (domain->glx == 2)
1830                free_gcr3_tbl_level2(domain->gcr3_tbl);
1831        else if (domain->glx == 1)
1832                free_gcr3_tbl_level1(domain->gcr3_tbl);
1833        else if (domain->glx != 0)
1834                BUG();
1835
1836        free_page((unsigned long)domain->gcr3_tbl);
1837}
1838
1839/*
1840 * Free a domain, only used if something went wrong in the
1841 * allocation path and we need to free an already allocated page table
1842 */
1843static void dma_ops_domain_free(struct dma_ops_domain *dom)
1844{
1845        int i;
1846
1847        if (!dom)
1848                return;
1849
1850        del_domain_from_list(&dom->domain);
1851
1852        free_pagetable(&dom->domain);
1853
1854        for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1855                if (!dom->aperture[i])
1856                        continue;
1857                free_page((unsigned long)dom->aperture[i]->bitmap);
1858                kfree(dom->aperture[i]);
1859        }
1860
1861        kfree(dom);
1862}
1863
1864/*
1865 * Allocates a new protection domain usable for the dma_ops functions.
1866 * It also initializes the page table and the address allocator data
1867 * structures required for the dma_ops interface
1868 */
1869static struct dma_ops_domain *dma_ops_domain_alloc(void)
1870{
1871        struct dma_ops_domain *dma_dom;
1872
1873        dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
1874        if (!dma_dom)
1875                return NULL;
1876
1877        spin_lock_init(&dma_dom->domain.lock);
1878
1879        dma_dom->domain.id = domain_id_alloc();
1880        if (dma_dom->domain.id == 0)
1881                goto free_dma_dom;
1882        INIT_LIST_HEAD(&dma_dom->domain.dev_list);
1883        dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1884        dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1885        dma_dom->domain.flags = PD_DMA_OPS_MASK;
1886        dma_dom->domain.priv = dma_dom;
1887        if (!dma_dom->domain.pt_root)
1888                goto free_dma_dom;
1889
1890        dma_dom->need_flush = false;
1891        dma_dom->target_dev = 0xffff;
1892
1893        add_domain_to_list(&dma_dom->domain);
1894
1895        if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1896                goto free_dma_dom;
1897
1898        /*
1899         * mark the first page as allocated so we never return 0 as
1900         * a valid dma-address. So we can use 0 as error value
1901         */
1902        dma_dom->aperture[0]->bitmap[0] = 1;
1903        dma_dom->next_address = 0;
1904
1905
1906        return dma_dom;
1907
1908free_dma_dom:
1909        dma_ops_domain_free(dma_dom);
1910
1911        return NULL;
1912}
1913
1914/*
1915 * little helper function to check whether a given protection domain is a
1916 * dma_ops domain
1917 */
1918static bool dma_ops_domain(struct protection_domain *domain)
1919{
1920        return domain->flags & PD_DMA_OPS_MASK;
1921}
1922
1923static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
1924{
1925        u64 pte_root = 0;
1926        u64 flags = 0;
1927
1928        if (domain->mode != PAGE_MODE_NONE)
1929                pte_root = virt_to_phys(domain->pt_root);
1930
1931        pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
1932                    << DEV_ENTRY_MODE_SHIFT;
1933        pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1934
1935        flags = amd_iommu_dev_table[devid].data[1];
1936
1937        if (ats)
1938                flags |= DTE_FLAG_IOTLB;
1939
1940        if (domain->flags & PD_IOMMUV2_MASK) {
1941                u64 gcr3 = __pa(domain->gcr3_tbl);
1942                u64 glx  = domain->glx;
1943                u64 tmp;
1944
1945                pte_root |= DTE_FLAG_GV;
1946                pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
1947
1948                /* First mask out possible old values for GCR3 table */
1949                tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1950                flags    &= ~tmp;
1951
1952                tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1953                flags    &= ~tmp;
1954
1955                /* Encode GCR3 table into DTE */
1956                tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
1957                pte_root |= tmp;
1958
1959                tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
1960                flags    |= tmp;
1961
1962                tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
1963                flags    |= tmp;
1964        }
1965
1966        flags &= ~(0xffffUL);
1967        flags |= domain->id;
1968
1969        amd_iommu_dev_table[devid].data[1]  = flags;
1970        amd_iommu_dev_table[devid].data[0]  = pte_root;
1971}
1972
1973static void clear_dte_entry(u16 devid)
1974{
1975        /* remove entry from the device table seen by the hardware */
1976        amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
1977        amd_iommu_dev_table[devid].data[1] = 0;
1978
1979        amd_iommu_apply_erratum_63(devid);
1980}
1981
1982static void do_attach(struct iommu_dev_data *dev_data,
1983                      struct protection_domain *domain)
1984{
1985        struct amd_iommu *iommu;
1986        bool ats;
1987
1988        iommu = amd_iommu_rlookup_table[dev_data->devid];
1989        ats   = dev_data->ats.enabled;
1990
1991        /* Update data structures */
1992        dev_data->domain = domain;
1993        list_add(&dev_data->list, &domain->dev_list);
1994        set_dte_entry(dev_data->devid, domain, ats);
1995
1996        /* Do reference counting */
1997        domain->dev_iommu[iommu->index] += 1;
1998        domain->dev_cnt                 += 1;
1999
2000        /* Flush the DTE entry */
2001        device_flush_dte(dev_data);
2002}
2003
2004static void do_detach(struct iommu_dev_data *dev_data)
2005{
2006        struct amd_iommu *iommu;
2007
2008        iommu = amd_iommu_rlookup_table[dev_data->devid];
2009
2010        /* decrease reference counters */
2011        dev_data->domain->dev_iommu[iommu->index] -= 1;
2012        dev_data->domain->dev_cnt                 -= 1;
2013
2014        /* Update data structures */
2015        dev_data->domain = NULL;
2016        list_del(&dev_data->list);
2017        clear_dte_entry(dev_data->devid);
2018
2019        /* Flush the DTE entry */
2020        device_flush_dte(dev_data);
2021}
2022
2023/*
2024 * If a device is not yet associated with a domain, this function does
2025 * assigns it visible for the hardware
2026 */
2027static int __attach_device(struct iommu_dev_data *dev_data,
2028                           struct protection_domain *domain)
2029{
2030        int ret;
2031
2032        /* lock domain */
2033        spin_lock(&domain->lock);
2034
2035        if (dev_data->alias_data != NULL) {
2036                struct iommu_dev_data *alias_data = dev_data->alias_data;
2037
2038                /* Some sanity checks */
2039                ret = -EBUSY;
2040                if (alias_data->domain != NULL &&
2041                                alias_data->domain != domain)
2042                        goto out_unlock;
2043
2044                if (dev_data->domain != NULL &&
2045                                dev_data->domain != domain)
2046                        goto out_unlock;
2047
2048                /* Do real assignment */
2049                if (alias_data->domain == NULL)
2050                        do_attach(alias_data, domain);
2051
2052                atomic_inc(&alias_data->bind);
2053        }
2054
2055        if (dev_data->domain == NULL)
2056                do_attach(dev_data, domain);
2057
2058        atomic_inc(&dev_data->bind);
2059
2060        ret = 0;
2061
2062out_unlock:
2063
2064        /* ready */
2065        spin_unlock(&domain->lock);
2066
2067        return ret;
2068}
2069
2070
2071static void pdev_iommuv2_disable(struct pci_dev *pdev)
2072{
2073        pci_disable_ats(pdev);
2074        pci_disable_pri(pdev);
2075        pci_disable_pasid(pdev);
2076}
2077
2078/* FIXME: Change generic reset-function to do the same */
2079static int pri_reset_while_enabled(struct pci_dev *pdev)
2080{
2081        u16 control;
2082        int pos;
2083
2084        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2085        if (!pos)
2086                return -EINVAL;
2087
2088        pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
2089        control |= PCI_PRI_CTRL_RESET;
2090        pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2091
2092        return 0;
2093}
2094
2095static int pdev_iommuv2_enable(struct pci_dev *pdev)
2096{
2097        bool reset_enable;
2098        int reqs, ret;
2099
2100        /* FIXME: Hardcode number of outstanding requests for now */
2101        reqs = 32;
2102        if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
2103                reqs = 1;
2104        reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2105
2106        /* Only allow access to user-accessible pages */
2107        ret = pci_enable_pasid(pdev, 0);
2108        if (ret)
2109                goto out_err;
2110
2111        /* First reset the PRI state of the device */
2112        ret = pci_reset_pri(pdev);
2113        if (ret)
2114                goto out_err;
2115
2116        /* Enable PRI */
2117        ret = pci_enable_pri(pdev, reqs);
2118        if (ret)
2119                goto out_err;
2120
2121        if (reset_enable) {
2122                ret = pri_reset_while_enabled(pdev);
2123                if (ret)
2124                        goto out_err;
2125        }
2126
2127        ret = pci_enable_ats(pdev, PAGE_SHIFT);
2128        if (ret)
2129                goto out_err;
2130
2131        return 0;
2132
2133out_err:
2134        pci_disable_pri(pdev);
2135        pci_disable_pasid(pdev);
2136
2137        return ret;
2138}
2139
2140/* FIXME: Move this to PCI code */
2141#define PCI_PRI_TLP_OFF         (1 << 15)
2142
2143static bool pci_pri_tlp_required(struct pci_dev *pdev)
2144{
2145        u16 status;
2146        int pos;
2147
2148        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2149        if (!pos)
2150                return false;
2151
2152        pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2153
2154        return (status & PCI_PRI_TLP_OFF) ? true : false;
2155}
2156
2157/*
2158 * If a device is not yet associated with a domain, this function does
2159 * assigns it visible for the hardware
2160 */
2161static int attach_device(struct device *dev,
2162                         struct protection_domain *domain)
2163{
2164        struct pci_dev *pdev = to_pci_dev(dev);
2165        struct iommu_dev_data *dev_data;
2166        unsigned long flags;
2167        int ret;
2168
2169        dev_data = get_dev_data(dev);
2170
2171        if (domain->flags & PD_IOMMUV2_MASK) {
2172                if (!dev_data->iommu_v2 || !dev_data->passthrough)
2173                        return -EINVAL;
2174
2175                if (pdev_iommuv2_enable(pdev) != 0)
2176                        return -EINVAL;
2177
2178                dev_data->ats.enabled = true;
2179                dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
2180                dev_data->pri_tlp     = pci_pri_tlp_required(pdev);
2181        } else if (amd_iommu_iotlb_sup &&
2182                   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2183                dev_data->ats.enabled = true;
2184                dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
2185        }
2186
2187        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2188        ret = __attach_device(dev_data, domain);
2189        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2190
2191        /*
2192         * We might boot into a crash-kernel here. The crashed kernel
2193         * left the caches in the IOMMU dirty. So we have to flush
2194         * here to evict all dirty stuff.
2195         */
2196        domain_flush_tlb_pde(domain);
2197
2198        return ret;
2199}
2200
2201/*
2202 * Removes a device from a protection domain (unlocked)
2203 */
2204static void __detach_device(struct iommu_dev_data *dev_data)
2205{
2206        struct protection_domain *domain;
2207        unsigned long flags;
2208
2209        BUG_ON(!dev_data->domain);
2210
2211        domain = dev_data->domain;
2212
2213        spin_lock_irqsave(&domain->lock, flags);
2214
2215        if (dev_data->alias_data != NULL) {
2216                struct iommu_dev_data *alias_data = dev_data->alias_data;
2217
2218                if (atomic_dec_and_test(&alias_data->bind))
2219                        do_detach(alias_data);
2220        }
2221
2222        if (atomic_dec_and_test(&dev_data->bind))
2223                do_detach(dev_data);
2224
2225        spin_unlock_irqrestore(&domain->lock, flags);
2226
2227        /*
2228         * If we run in passthrough mode the device must be assigned to the
2229         * passthrough domain if it is detached from any other domain.
2230         * Make sure we can deassign from the pt_domain itself.
2231         */
2232        if (dev_data->passthrough &&
2233            (dev_data->domain == NULL && domain != pt_domain))
2234                __attach_device(dev_data, pt_domain);
2235}
2236
2237/*
2238 * Removes a device from a protection domain (with devtable_lock held)
2239 */
2240static void detach_device(struct device *dev)
2241{
2242        struct protection_domain *domain;
2243        struct iommu_dev_data *dev_data;
2244        unsigned long flags;
2245
2246        dev_data = get_dev_data(dev);
2247        domain   = dev_data->domain;
2248
2249        /* lock device table */
2250        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2251        __detach_device(dev_data);
2252        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2253
2254        if (domain->flags & PD_IOMMUV2_MASK)
2255                pdev_iommuv2_disable(to_pci_dev(dev));
2256        else if (dev_data->ats.enabled)
2257                pci_disable_ats(to_pci_dev(dev));
2258
2259        dev_data->ats.enabled = false;
2260}
2261
2262/*
2263 * Find out the protection domain structure for a given PCI device. This
2264 * will give us the pointer to the page table root for example.
2265 */
2266static struct protection_domain *domain_for_device(struct device *dev)
2267{
2268        struct iommu_dev_data *dev_data;
2269        struct protection_domain *dom = NULL;
2270        unsigned long flags;
2271
2272        dev_data   = get_dev_data(dev);
2273
2274        if (dev_data->domain)
2275                return dev_data->domain;
2276
2277        if (dev_data->alias_data != NULL) {
2278                struct iommu_dev_data *alias_data = dev_data->alias_data;
2279
2280                read_lock_irqsave(&amd_iommu_devtable_lock, flags);
2281                if (alias_data->domain != NULL) {
2282                        __attach_device(dev_data, alias_data->domain);
2283                        dom = alias_data->domain;
2284                }
2285                read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2286        }
2287
2288        return dom;
2289}
2290
2291static int device_change_notifier(struct notifier_block *nb,
2292                                  unsigned long action, void *data)
2293{
2294        struct dma_ops_domain *dma_domain;
2295        struct protection_domain *domain;
2296        struct iommu_dev_data *dev_data;
2297        struct device *dev = data;
2298        struct amd_iommu *iommu;
2299        unsigned long flags;
2300        u16 devid;
2301
2302        if (!check_device(dev))
2303                return 0;
2304
2305        devid    = get_device_id(dev);
2306        iommu    = amd_iommu_rlookup_table[devid];
2307        dev_data = get_dev_data(dev);
2308
2309        switch (action) {
2310        case BUS_NOTIFY_UNBOUND_DRIVER:
2311
2312                domain = domain_for_device(dev);
2313
2314                if (!domain)
2315                        goto out;
2316                if (dev_data->passthrough)
2317                        break;
2318                detach_device(dev);
2319                break;
2320        case BUS_NOTIFY_ADD_DEVICE:
2321
2322                iommu_init_device(dev);
2323
2324                /*
2325                 * dev_data is still NULL and
2326                 * got initialized in iommu_init_device
2327                 */
2328                dev_data = get_dev_data(dev);
2329
2330                if (iommu_pass_through || dev_data->iommu_v2) {
2331                        dev_data->passthrough = true;
2332                        attach_device(dev, pt_domain);
2333                        break;
2334                }
2335
2336                domain = domain_for_device(dev);
2337
2338                /* allocate a protection domain if a device is added */
2339                dma_domain = find_protection_domain(devid);
2340                if (dma_domain)
2341                        goto out;
2342                dma_domain = dma_ops_domain_alloc();
2343                if (!dma_domain)
2344                        goto out;
2345                dma_domain->target_dev = devid;
2346
2347                spin_lock_irqsave(&iommu_pd_list_lock, flags);
2348                list_add_tail(&dma_domain->list, &iommu_pd_list);
2349                spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
2350
2351                dev_data = get_dev_data(dev);
2352
2353                dev->archdata.dma_ops = &amd_iommu_dma_ops;
2354
2355                break;
2356        case BUS_NOTIFY_DEL_DEVICE:
2357
2358                iommu_uninit_device(dev);
2359
2360        default:
2361                goto out;
2362        }
2363
2364        iommu_completion_wait(iommu);
2365
2366out:
2367        return 0;
2368}
2369
2370static struct notifier_block device_nb = {
2371        .notifier_call = device_change_notifier,
2372};
2373
2374void amd_iommu_init_notifier(void)
2375{
2376        bus_register_notifier(&pci_bus_type, &device_nb);
2377}
2378
2379/*****************************************************************************
2380 *
2381 * The next functions belong to the dma_ops mapping/unmapping code.
2382 *
2383 *****************************************************************************/
2384
2385/*
2386 * In the dma_ops path we only have the struct device. This function
2387 * finds the corresponding IOMMU, the protection domain and the
2388 * requestor id for a given device.
2389 * If the device is not yet associated with a domain this is also done
2390 * in this function.
2391 */
2392static struct protection_domain *get_domain(struct device *dev)
2393{
2394        struct protection_domain *domain;
2395        struct dma_ops_domain *dma_dom;
2396        u16 devid = get_device_id(dev);
2397
2398        if (!check_device(dev))
2399                return ERR_PTR(-EINVAL);
2400
2401        domain = domain_for_device(dev);
2402        if (domain != NULL && !dma_ops_domain(domain))
2403                return ERR_PTR(-EBUSY);
2404
2405        if (domain != NULL)
2406                return domain;
2407
2408        /* Device not bount yet - bind it */
2409        dma_dom = find_protection_domain(devid);
2410        if (!dma_dom)
2411                dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
2412        attach_device(dev, &dma_dom->domain);
2413        DUMP_printk("Using protection domain %d for device %s\n",
2414                    dma_dom->domain.id, dev_name(dev));
2415
2416        return &dma_dom->domain;
2417}
2418
2419static void update_device_table(struct protection_domain *domain)
2420{
2421        struct iommu_dev_data *dev_data;
2422
2423        list_for_each_entry(dev_data, &domain->dev_list, list)
2424                set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2425}
2426
2427static void update_domain(struct protection_domain *domain)
2428{
2429        if (!domain->updated)
2430                return;
2431
2432        update_device_table(domain);
2433
2434        domain_flush_devices(domain);
2435        domain_flush_tlb_pde(domain);
2436
2437        domain->updated = false;
2438}
2439
2440/*
2441 * This function fetches the PTE for a given address in the aperture
2442 */
2443static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
2444                            unsigned long address)
2445{
2446        struct aperture_range *aperture;
2447        u64 *pte, *pte_page;
2448
2449        aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2450        if (!aperture)
2451                return NULL;
2452
2453        pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2454        if (!pte) {
2455                pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2456                                GFP_ATOMIC);
2457                aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
2458        } else
2459                pte += PM_LEVEL_INDEX(0, address);
2460
2461        update_domain(&dom->domain);
2462
2463        return pte;
2464}
2465
2466/*
2467 * This is the generic map function. It maps one 4kb page at paddr to
2468 * the given address in the DMA address space for the domain.
2469 */
2470static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2471                                     unsigned long address,
2472                                     phys_addr_t paddr,
2473                                     int direction)
2474{
2475        u64 *pte, __pte;
2476
2477        WARN_ON(address > dom->aperture_size);
2478
2479        paddr &= PAGE_MASK;
2480
2481        pte  = dma_ops_get_pte(dom, address);
2482        if (!pte)
2483                return DMA_ERROR_CODE;
2484
2485        __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
2486
2487        if (direction == DMA_TO_DEVICE)
2488                __pte |= IOMMU_PTE_IR;
2489        else if (direction == DMA_FROM_DEVICE)
2490                __pte |= IOMMU_PTE_IW;
2491        else if (direction == DMA_BIDIRECTIONAL)
2492                __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
2493
2494        WARN_ON(*pte);
2495
2496        *pte = __pte;
2497
2498        return (dma_addr_t)address;
2499}
2500
2501/*
2502 * The generic unmapping function for on page in the DMA address space.
2503 */
2504static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2505                                 unsigned long address)
2506{
2507        struct aperture_range *aperture;
2508        u64 *pte;
2509
2510        if (address >= dom->aperture_size)
2511                return;
2512
2513        aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2514        if (!aperture)
2515                return;
2516
2517        pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2518        if (!pte)
2519                return;
2520
2521        pte += PM_LEVEL_INDEX(0, address);
2522
2523        WARN_ON(!*pte);
2524
2525        *pte = 0ULL;
2526}
2527
2528/*
2529 * This function contains common code for mapping of a physically
2530 * contiguous memory region into DMA address space. It is used by all
2531 * mapping functions provided with this IOMMU driver.
2532 * Must be called with the domain lock held.
2533 */
2534static dma_addr_t __map_single(struct device *dev,
2535                               struct dma_ops_domain *dma_dom,
2536                               phys_addr_t paddr,
2537                               size_t size,
2538                               int dir,
2539                               bool align,
2540                               u64 dma_mask)
2541{
2542        dma_addr_t offset = paddr & ~PAGE_MASK;
2543        dma_addr_t address, start, ret;
2544        unsigned int pages;
2545        unsigned long align_mask = 0;
2546        int i;
2547
2548        pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2549        paddr &= PAGE_MASK;
2550
2551        INC_STATS_COUNTER(total_map_requests);
2552
2553        if (pages > 1)
2554                INC_STATS_COUNTER(cross_page);
2555
2556        if (align)
2557                align_mask = (1UL << get_order(size)) - 1;
2558
2559retry:
2560        address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
2561                                          dma_mask);
2562        if (unlikely(address == DMA_ERROR_CODE)) {
2563                /*
2564                 * setting next_address here will let the address
2565                 * allocator only scan the new allocated range in the
2566                 * first run. This is a small optimization.
2567                 */
2568                dma_dom->next_address = dma_dom->aperture_size;
2569
2570                if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2571                        goto out;
2572
2573                /*
2574                 * aperture was successfully enlarged by 128 MB, try
2575                 * allocation again
2576                 */
2577                goto retry;
2578        }
2579
2580        start = address;
2581        for (i = 0; i < pages; ++i) {
2582                ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2583                if (ret == DMA_ERROR_CODE)
2584                        goto out_unmap;
2585
2586                paddr += PAGE_SIZE;
2587                start += PAGE_SIZE;
2588        }
2589        address += offset;
2590
2591        ADD_STATS_COUNTER(alloced_io_mem, size);
2592
2593        if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2594                domain_flush_tlb(&dma_dom->domain);
2595                dma_dom->need_flush = false;
2596        } else if (unlikely(amd_iommu_np_cache))
2597                domain_flush_pages(&dma_dom->domain, address, size);
2598
2599out:
2600        return address;
2601
2602out_unmap:
2603
2604        for (--i; i >= 0; --i) {
2605                start -= PAGE_SIZE;
2606                dma_ops_domain_unmap(dma_dom, start);
2607        }
2608
2609        dma_ops_free_addresses(dma_dom, address, pages);
2610
2611        return DMA_ERROR_CODE;
2612}
2613
2614/*
2615 * Does the reverse of the __map_single function. Must be called with
2616 * the domain lock held too
2617 */
2618static void __unmap_single(struct dma_ops_domain *dma_dom,
2619                           dma_addr_t dma_addr,
2620                           size_t size,
2621                           int dir)
2622{
2623        dma_addr_t flush_addr;
2624        dma_addr_t i, start;
2625        unsigned int pages;
2626
2627        if ((dma_addr == DMA_ERROR_CODE) ||
2628            (dma_addr + size > dma_dom->aperture_size))
2629                return;
2630
2631        flush_addr = dma_addr;
2632        pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2633        dma_addr &= PAGE_MASK;
2634        start = dma_addr;
2635
2636        for (i = 0; i < pages; ++i) {
2637                dma_ops_domain_unmap(dma_dom, start);
2638                start += PAGE_SIZE;
2639        }
2640
2641        SUB_STATS_COUNTER(alloced_io_mem, size);
2642
2643        dma_ops_free_addresses(dma_dom, dma_addr, pages);
2644
2645        if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2646                domain_flush_pages(&dma_dom->domain, flush_addr, size);
2647                dma_dom->need_flush = false;
2648        }
2649}
2650
2651/*
2652 * The exported map_single function for dma_ops.
2653 */
2654static dma_addr_t map_page(struct device *dev, struct page *page,
2655                           unsigned long offset, size_t size,
2656                           enum dma_data_direction dir,
2657                           struct dma_attrs *attrs)
2658{
2659        unsigned long flags;
2660        struct protection_domain *domain;
2661        dma_addr_t addr;
2662        u64 dma_mask;
2663        phys_addr_t paddr = page_to_phys(page) + offset;
2664
2665        INC_STATS_COUNTER(cnt_map_single);
2666
2667        domain = get_domain(dev);
2668        if (PTR_ERR(domain) == -EINVAL)
2669                return (dma_addr_t)paddr;
2670        else if (IS_ERR(domain))
2671                return DMA_ERROR_CODE;
2672
2673        dma_mask = *dev->dma_mask;
2674
2675        spin_lock_irqsave(&domain->lock, flags);
2676
2677        addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2678                            dma_mask);
2679        if (addr == DMA_ERROR_CODE)
2680                goto out;
2681
2682        domain_flush_complete(domain);
2683
2684out:
2685        spin_unlock_irqrestore(&domain->lock, flags);
2686
2687        return addr;
2688}
2689
2690/*
2691 * The exported unmap_single function for dma_ops.
2692 */
2693static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2694                       enum dma_data_direction dir, struct dma_attrs *attrs)
2695{
2696        unsigned long flags;
2697        struct protection_domain *domain;
2698
2699        INC_STATS_COUNTER(cnt_unmap_single);
2700
2701        domain = get_domain(dev);
2702        if (IS_ERR(domain))
2703                return;
2704
2705        spin_lock_irqsave(&domain->lock, flags);
2706
2707        __unmap_single(domain->priv, dma_addr, size, dir);
2708
2709        domain_flush_complete(domain);
2710
2711        spin_unlock_irqrestore(&domain->lock, flags);
2712}
2713
2714/*
2715 * This is a special map_sg function which is used if we should map a
2716 * device which is not handled by an AMD IOMMU in the system.
2717 */
2718static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
2719                           int nelems, int dir)
2720{
2721        struct scatterlist *s;
2722        int i;
2723
2724        for_each_sg(sglist, s, nelems, i) {
2725                s->dma_address = (dma_addr_t)sg_phys(s);
2726                s->dma_length  = s->length;
2727        }
2728
2729        return nelems;
2730}
2731
2732/*
2733 * The exported map_sg function for dma_ops (handles scatter-gather
2734 * lists).
2735 */
2736static int map_sg(struct device *dev, struct scatterlist *sglist,
2737                  int nelems, enum dma_data_direction dir,
2738                  struct dma_attrs *attrs)
2739{
2740        unsigned long flags;
2741        struct protection_domain *domain;
2742        int i;
2743        struct scatterlist *s;
2744        phys_addr_t paddr;
2745        int mapped_elems = 0;
2746        u64 dma_mask;
2747
2748        INC_STATS_COUNTER(cnt_map_sg);
2749
2750        domain = get_domain(dev);
2751        if (PTR_ERR(domain) == -EINVAL)
2752                return map_sg_no_iommu(dev, sglist, nelems, dir);
2753        else if (IS_ERR(domain))
2754                return 0;
2755
2756        dma_mask = *dev->dma_mask;
2757
2758        spin_lock_irqsave(&domain->lock, flags);
2759
2760        for_each_sg(sglist, s, nelems, i) {
2761                paddr = sg_phys(s);
2762
2763                s->dma_address = __map_single(dev, domain->priv,
2764                                              paddr, s->length, dir, false,
2765                                              dma_mask);
2766
2767                if (s->dma_address) {
2768                        s->dma_length = s->length;
2769                        mapped_elems++;
2770                } else
2771                        goto unmap;
2772        }
2773
2774        domain_flush_complete(domain);
2775
2776out:
2777        spin_unlock_irqrestore(&domain->lock, flags);
2778
2779        return mapped_elems;
2780unmap:
2781        for_each_sg(sglist, s, mapped_elems, i) {
2782                if (s->dma_address)
2783                        __unmap_single(domain->priv, s->dma_address,
2784                                       s->dma_length, dir);
2785                s->dma_address = s->dma_length = 0;
2786        }
2787
2788        mapped_elems = 0;
2789
2790        goto out;
2791}
2792
2793/*
2794 * The exported map_sg function for dma_ops (handles scatter-gather
2795 * lists).
2796 */
2797static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2798                     int nelems, enum dma_data_direction dir,
2799                     struct dma_attrs *attrs)
2800{
2801        unsigned long flags;
2802        struct protection_domain *domain;
2803        struct scatterlist *s;
2804        int i;
2805
2806        INC_STATS_COUNTER(cnt_unmap_sg);
2807
2808        domain = get_domain(dev);
2809        if (IS_ERR(domain))
2810                return;
2811
2812        spin_lock_irqsave(&domain->lock, flags);
2813
2814        for_each_sg(sglist, s, nelems, i) {
2815                __unmap_single(domain->priv, s->dma_address,
2816                               s->dma_length, dir);
2817                s->dma_address = s->dma_length = 0;
2818        }
2819
2820        domain_flush_complete(domain);
2821
2822        spin_unlock_irqrestore(&domain->lock, flags);
2823}
2824
2825/*
2826 * The exported alloc_coherent function for dma_ops.
2827 */
2828static void *alloc_coherent(struct device *dev, size_t size,
2829                            dma_addr_t *dma_addr, gfp_t flag,
2830                            struct dma_attrs *attrs)
2831{
2832        unsigned long flags;
2833        void *virt_addr;
2834        struct protection_domain *domain;
2835        phys_addr_t paddr;
2836        u64 dma_mask = dev->coherent_dma_mask;
2837
2838        INC_STATS_COUNTER(cnt_alloc_coherent);
2839
2840        domain = get_domain(dev);
2841        if (PTR_ERR(domain) == -EINVAL) {
2842                virt_addr = (void *)__get_free_pages(flag, get_order(size));
2843                *dma_addr = __pa(virt_addr);
2844                return virt_addr;
2845        } else if (IS_ERR(domain))
2846                return NULL;
2847
2848        dma_mask  = dev->coherent_dma_mask;
2849        flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2850        flag     |= __GFP_ZERO;
2851
2852        virt_addr = (void *)__get_free_pages(flag, get_order(size));
2853        if (!virt_addr)
2854                return NULL;
2855
2856        paddr = virt_to_phys(virt_addr);
2857
2858        if (!dma_mask)
2859                dma_mask = *dev->dma_mask;
2860
2861        spin_lock_irqsave(&domain->lock, flags);
2862
2863        *dma_addr = __map_single(dev, domain->priv, paddr,
2864                                 size, DMA_BIDIRECTIONAL, true, dma_mask);
2865
2866        if (*dma_addr == DMA_ERROR_CODE) {
2867                spin_unlock_irqrestore(&domain->lock, flags);
2868                goto out_free;
2869        }
2870
2871        domain_flush_complete(domain);
2872
2873        spin_unlock_irqrestore(&domain->lock, flags);
2874
2875        return virt_addr;
2876
2877out_free:
2878
2879        free_pages((unsigned long)virt_addr, get_order(size));
2880
2881        return NULL;
2882}
2883
2884/*
2885 * The exported free_coherent function for dma_ops.
2886 */
2887static void free_coherent(struct device *dev, size_t size,
2888                          void *virt_addr, dma_addr_t dma_addr,
2889                          struct dma_attrs *attrs)
2890{
2891        unsigned long flags;
2892        struct protection_domain *domain;
2893
2894        INC_STATS_COUNTER(cnt_free_coherent);
2895
2896        domain = get_domain(dev);
2897        if (IS_ERR(domain))
2898                goto free_mem;
2899
2900        spin_lock_irqsave(&domain->lock, flags);
2901
2902        __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2903
2904        domain_flush_complete(domain);
2905
2906        spin_unlock_irqrestore(&domain->lock, flags);
2907
2908free_mem:
2909        free_pages((unsigned long)virt_addr, get_order(size));
2910}
2911
2912/*
2913 * This function is called by the DMA layer to find out if we can handle a
2914 * particular device. It is part of the dma_ops.
2915 */
2916static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2917{
2918        return check_device(dev);
2919}
2920
2921/*
2922 * The function for pre-allocating protection domains.
2923 *
2924 * If the driver core informs the DMA layer if a driver grabs a device
2925 * we don't need to preallocate the protection domains anymore.
2926 * For now we have to.
2927 */
2928static void __init prealloc_protection_domains(void)
2929{
2930        struct iommu_dev_data *dev_data;
2931        struct dma_ops_domain *dma_dom;
2932        struct pci_dev *dev = NULL;
2933        u16 devid;
2934
2935        for_each_pci_dev(dev) {
2936
2937                /* Do we handle this device? */
2938                if (!check_device(&dev->dev))
2939                        continue;
2940
2941                dev_data = get_dev_data(&dev->dev);
2942                if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
2943                        /* Make sure passthrough domain is allocated */
2944                        alloc_passthrough_domain();
2945                        dev_data->passthrough = true;
2946                        attach_device(&dev->dev, pt_domain);
2947                        pr_info("AMD-Vi: Using passthough domain for device %s\n",
2948                                dev_name(&dev->dev));
2949                }
2950
2951                /* Is there already any domain for it? */
2952                if (domain_for_device(&dev->dev))
2953                        continue;
2954
2955                devid = get_device_id(&dev->dev);
2956
2957                dma_dom = dma_ops_domain_alloc();
2958                if (!dma_dom)
2959                        continue;
2960                init_unity_mappings_for_device(dma_dom, devid);
2961                dma_dom->target_dev = devid;
2962
2963                attach_device(&dev->dev, &dma_dom->domain);
2964
2965                list_add_tail(&dma_dom->list, &iommu_pd_list);
2966        }
2967}
2968
2969static struct dma_map_ops amd_iommu_dma_ops = {
2970        .alloc = alloc_coherent,
2971        .free = free_coherent,
2972        .map_page = map_page,
2973        .unmap_page = unmap_page,
2974        .map_sg = map_sg,
2975        .unmap_sg = unmap_sg,
2976        .dma_supported = amd_iommu_dma_supported,
2977};
2978
2979static unsigned device_dma_ops_init(void)
2980{
2981        struct iommu_dev_data *dev_data;
2982        struct pci_dev *pdev = NULL;
2983        unsigned unhandled = 0;
2984
2985        for_each_pci_dev(pdev) {
2986                if (!check_device(&pdev->dev)) {
2987
2988                        iommu_ignore_device(&pdev->dev);
2989
2990                        unhandled += 1;
2991                        continue;
2992                }
2993
2994                dev_data = get_dev_data(&pdev->dev);
2995
2996                if (!dev_data->passthrough)
2997                        pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
2998                else
2999                        pdev->dev.archdata.dma_ops = &nommu_dma_ops;
3000        }
3001
3002        return unhandled;
3003}
3004
3005/*
3006 * The function which clues the AMD IOMMU driver into dma_ops.
3007 */
3008
3009void __init amd_iommu_init_api(void)
3010{
3011        bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
3012}
3013
3014int __init amd_iommu_init_dma_ops(void)
3015{
3016        struct amd_iommu *iommu;
3017        int ret, unhandled;
3018
3019        /*
3020         * first allocate a default protection domain for every IOMMU we
3021         * found in the system. Devices not assigned to any other
3022         * protection domain will be assigned to the default one.
3023         */
3024        for_each_iommu(iommu) {
3025                iommu->default_dom = dma_ops_domain_alloc();
3026                if (iommu->default_dom == NULL)
3027                        return -ENOMEM;
3028                iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
3029                ret = iommu_init_unity_mappings(iommu);
3030                if (ret)
3031                        goto free_domains;
3032        }
3033
3034        /*
3035         * Pre-allocate the protection domains for each device.
3036         */
3037        prealloc_protection_domains();
3038
3039        iommu_detected = 1;
3040        swiotlb = 0;
3041
3042        /* Make the driver finally visible to the drivers */
3043        unhandled = device_dma_ops_init();
3044        if (unhandled && max_pfn > MAX_DMA32_PFN) {
3045                /* There are unhandled devices - initialize swiotlb for them */
3046                swiotlb = 1;
3047        }
3048
3049        amd_iommu_stats_init();
3050
3051        if (amd_iommu_unmap_flush)
3052                pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
3053        else
3054                pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
3055
3056        return 0;
3057
3058free_domains:
3059
3060        for_each_iommu(iommu) {
3061                if (iommu->default_dom)
3062                        dma_ops_domain_free(iommu->default_dom);
3063        }
3064
3065        return ret;
3066}
3067
3068/*****************************************************************************
3069 *
3070 * The following functions belong to the exported interface of AMD IOMMU
3071 *
3072 * This interface allows access to lower level functions of the IOMMU
3073 * like protection domain handling and assignement of devices to domains
3074 * which is not possible with the dma_ops interface.
3075 *
3076 *****************************************************************************/
3077
3078static void cleanup_domain(struct protection_domain *domain)
3079{
3080        struct iommu_dev_data *dev_data, *next;
3081        unsigned long flags;
3082
3083        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3084
3085        list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
3086                __detach_device(dev_data);
3087                atomic_set(&dev_data->bind, 0);
3088        }
3089
3090        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
3091}
3092
3093static void protection_domain_free(struct protection_domain *domain)
3094{
3095        if (!domain)
3096                return;
3097
3098        del_domain_from_list(domain);
3099
3100        if (domain->id)
3101                domain_id_free(domain->id);
3102
3103        kfree(domain);
3104}
3105
3106static struct protection_domain *protection_domain_alloc(void)
3107{
3108        struct protection_domain *domain;
3109
3110        domain = kzalloc(sizeof(*domain), GFP_KERNEL);
3111        if (!domain)
3112                return NULL;
3113
3114        spin_lock_init(&domain->lock);
3115        mutex_init(&domain->api_lock);
3116        domain->id = domain_id_alloc();
3117        if (!domain->id)
3118                goto out_err;
3119        INIT_LIST_HEAD(&domain->dev_list);
3120
3121        add_domain_to_list(domain);
3122
3123        return domain;
3124
3125out_err:
3126        kfree(domain);
3127
3128        return NULL;
3129}
3130
3131static int __init alloc_passthrough_domain(void)
3132{
3133        if (pt_domain != NULL)
3134                return 0;
3135
3136        /* allocate passthrough domain */
3137        pt_domain = protection_domain_alloc();
3138        if (!pt_domain)
3139                return -ENOMEM;
3140
3141        pt_domain->mode = PAGE_MODE_NONE;
3142
3143        return 0;
3144}
3145static int amd_iommu_domain_init(struct iommu_domain *dom)
3146{
3147        struct protection_domain *domain;
3148
3149        domain = protection_domain_alloc();
3150        if (!domain)
3151                goto out_free;
3152
3153        domain->mode    = PAGE_MODE_3_LEVEL;
3154        domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
3155        if (!domain->pt_root)
3156                goto out_free;
3157
3158        domain->iommu_domain = dom;
3159
3160        dom->priv = domain;
3161
3162        dom->geometry.aperture_start = 0;
3163        dom->geometry.aperture_end   = ~0ULL;
3164        dom->geometry.force_aperture = true;
3165
3166        return 0;
3167
3168out_free:
3169        protection_domain_free(domain);
3170
3171        return -ENOMEM;
3172}
3173
3174static void amd_iommu_domain_destroy(struct iommu_domain *dom)
3175{
3176        struct protection_domain *domain = dom->priv;
3177
3178        if (!domain)
3179                return;
3180
3181        if (domain->dev_cnt > 0)
3182                cleanup_domain(domain);
3183
3184        BUG_ON(domain->dev_cnt != 0);
3185
3186        if (domain->mode != PAGE_MODE_NONE)
3187                free_pagetable(domain);
3188
3189        if (domain->flags & PD_IOMMUV2_MASK)
3190                free_gcr3_table(domain);
3191
3192        protection_domain_free(domain);
3193
3194        dom->priv = NULL;
3195}
3196
3197static void amd_iommu_detach_device(struct iommu_domain *dom,
3198                                    struct device *dev)
3199{
3200        struct iommu_dev_data *dev_data = dev->archdata.iommu;
3201        struct amd_iommu *iommu;
3202        u16 devid;
3203
3204        if (!check_device(dev))
3205                return;
3206
3207        devid = get_device_id(dev);
3208
3209        if (dev_data->domain != NULL)
3210                detach_device(dev);
3211
3212        iommu = amd_iommu_rlookup_table[devid];
3213        if (!iommu)
3214                return;
3215
3216        iommu_completion_wait(iommu);
3217}
3218
3219static int amd_iommu_attach_device(struct iommu_domain *dom,
3220                                   struct device *dev)
3221{
3222        struct protection_domain *domain = dom->priv;
3223        struct iommu_dev_data *dev_data;
3224        struct amd_iommu *iommu;
3225        int ret;
3226
3227        if (!check_device(dev))
3228                return -EINVAL;
3229
3230        dev_data = dev->archdata.iommu;
3231
3232        iommu = amd_iommu_rlookup_table[dev_data->devid];
3233        if (!iommu)
3234                return -EINVAL;
3235
3236        if (dev_data->domain)
3237                detach_device(dev);
3238
3239        ret = attach_device(dev, domain);
3240
3241        iommu_completion_wait(iommu);
3242
3243        return ret;
3244}
3245
3246static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3247                         phys_addr_t paddr, size_t page_size, int iommu_prot)
3248{
3249        struct protection_domain *domain = dom->priv;
3250        int prot = 0;
3251        int ret;
3252
3253        if (domain->mode == PAGE_MODE_NONE)
3254                return -EINVAL;
3255
3256        if (iommu_prot & IOMMU_READ)
3257                prot |= IOMMU_PROT_IR;
3258        if (iommu_prot & IOMMU_WRITE)
3259                prot |= IOMMU_PROT_IW;
3260
3261        mutex_lock(&domain->api_lock);
3262        ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3263        mutex_unlock(&domain->api_lock);
3264
3265        return ret;
3266}
3267
3268static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
3269                           size_t page_size)
3270{
3271        struct protection_domain *domain = dom->priv;
3272        size_t unmap_size;
3273
3274        if (domain->mode == PAGE_MODE_NONE)
3275                return -EINVAL;
3276
3277        mutex_lock(&domain->api_lock);
3278        unmap_size = iommu_unmap_page(domain, iova, page_size);
3279        mutex_unlock(&domain->api_lock);
3280
3281        domain_flush_tlb_pde(domain);
3282
3283        return unmap_size;
3284}
3285
3286static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3287                                          unsigned long iova)
3288{
3289        struct protection_domain *domain = dom->priv;
3290        unsigned long offset_mask;
3291        phys_addr_t paddr;
3292        u64 *pte, __pte;
3293
3294        if (domain->mode == PAGE_MODE_NONE)
3295                return iova;
3296
3297        pte = fetch_pte(domain, iova);
3298
3299        if (!pte || !IOMMU_PTE_PRESENT(*pte))
3300                return 0;
3301
3302        if (PM_PTE_LEVEL(*pte) == 0)
3303                offset_mask = PAGE_SIZE - 1;
3304        else
3305                offset_mask = PTE_PAGE_SIZE(*pte) - 1;
3306
3307        __pte = *pte & PM_ADDR_MASK;
3308        paddr = (__pte & ~offset_mask) | (iova & offset_mask);
3309
3310        return paddr;
3311}
3312
3313static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
3314                                    unsigned long cap)
3315{
3316        switch (cap) {
3317        case IOMMU_CAP_CACHE_COHERENCY:
3318                return 1;
3319        }
3320
3321        return 0;
3322}
3323
3324static struct iommu_ops amd_iommu_ops = {
3325        .domain_init = amd_iommu_domain_init,
3326        .domain_destroy = amd_iommu_domain_destroy,
3327        .attach_dev = amd_iommu_attach_device,
3328        .detach_dev = amd_iommu_detach_device,
3329        .map = amd_iommu_map,
3330        .unmap = amd_iommu_unmap,
3331        .iova_to_phys = amd_iommu_iova_to_phys,
3332        .domain_has_cap = amd_iommu_domain_has_cap,
3333        .pgsize_bitmap  = AMD_IOMMU_PGSIZES,
3334};
3335
3336/*****************************************************************************
3337 *
3338 * The next functions do a basic initialization of IOMMU for pass through
3339 * mode
3340 *
3341 * In passthrough mode the IOMMU is initialized and enabled but not used for
3342 * DMA-API translation.
3343 *
3344 *****************************************************************************/
3345
3346int __init amd_iommu_init_passthrough(void)
3347{
3348        struct iommu_dev_data *dev_data;
3349        struct pci_dev *dev = NULL;
3350        struct amd_iommu *iommu;
3351        u16 devid;
3352        int ret;
3353
3354        ret = alloc_passthrough_domain();
3355        if (ret)
3356                return ret;
3357
3358        for_each_pci_dev(dev) {
3359                if (!check_device(&dev->dev))
3360                        continue;
3361
3362                dev_data = get_dev_data(&dev->dev);
3363                dev_data->passthrough = true;
3364
3365                devid = get_device_id(&dev->dev);
3366
3367                iommu = amd_iommu_rlookup_table[devid];
3368                if (!iommu)
3369                        continue;
3370
3371                attach_device(&dev->dev, pt_domain);
3372        }
3373
3374        amd_iommu_stats_init();
3375
3376        pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
3377
3378        return 0;
3379}
3380
3381/* IOMMUv2 specific functions */
3382int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3383{
3384        return atomic_notifier_chain_register(&ppr_notifier, nb);
3385}
3386EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
3387
3388int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
3389{
3390        return atomic_notifier_chain_unregister(&ppr_notifier, nb);
3391}
3392EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3393
3394void amd_iommu_domain_direct_map(struct iommu_domain *dom)
3395{
3396        struct protection_domain *domain = dom->priv;
3397        unsigned long flags;
3398
3399        spin_lock_irqsave(&domain->lock, flags);
3400
3401        /* Update data structure */
3402        domain->mode    = PAGE_MODE_NONE;
3403        domain->updated = true;
3404
3405        /* Make changes visible to IOMMUs */
3406        update_domain(domain);
3407
3408        /* Page-table is not visible to IOMMU anymore, so free it */
3409        free_pagetable(domain);
3410
3411        spin_unlock_irqrestore(&domain->lock, flags);
3412}
3413EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3414
3415int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
3416{
3417        struct protection_domain *domain = dom->priv;
3418        unsigned long flags;
3419        int levels, ret;
3420
3421        if (pasids <= 0 || pasids > (PASID_MASK + 1))
3422                return -EINVAL;
3423
3424        /* Number of GCR3 table levels required */
3425        for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
3426                levels += 1;
3427
3428        if (levels > amd_iommu_max_glx_val)
3429                return -EINVAL;
3430
3431        spin_lock_irqsave(&domain->lock, flags);
3432
3433        /*
3434         * Save us all sanity checks whether devices already in the
3435         * domain support IOMMUv2. Just force that the domain has no
3436         * devices attached when it is switched into IOMMUv2 mode.
3437         */
3438        ret = -EBUSY;
3439        if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
3440                goto out;
3441
3442        ret = -ENOMEM;
3443        domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
3444        if (domain->gcr3_tbl == NULL)
3445                goto out;
3446
3447        domain->glx      = levels;
3448        domain->flags   |= PD_IOMMUV2_MASK;
3449        domain->updated  = true;
3450
3451        update_domain(domain);
3452
3453        ret = 0;
3454
3455out:
3456        spin_unlock_irqrestore(&domain->lock, flags);
3457
3458        return ret;
3459}
3460EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3461
3462static int __flush_pasid(struct protection_domain *domain, int pasid,
3463                         u64 address, bool size)
3464{
3465        struct iommu_dev_data *dev_data;
3466        struct iommu_cmd cmd;
3467        int i, ret;
3468
3469        if (!(domain->flags & PD_IOMMUV2_MASK))
3470                return -EINVAL;
3471
3472        build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
3473
3474        /*
3475         * IOMMU TLB needs to be flushed before Device TLB to
3476         * prevent device TLB refill from IOMMU TLB
3477         */
3478        for (i = 0; i < amd_iommus_present; ++i) {
3479                if (domain->dev_iommu[i] == 0)
3480                        continue;
3481
3482                ret = iommu_queue_command(amd_iommus[i], &cmd);
3483                if (ret != 0)
3484                        goto out;
3485        }
3486
3487        /* Wait until IOMMU TLB flushes are complete */
3488        domain_flush_complete(domain);
3489
3490        /* Now flush device TLBs */
3491        list_for_each_entry(dev_data, &domain->dev_list, list) {
3492                struct amd_iommu *iommu;
3493                int qdep;
3494
3495                BUG_ON(!dev_data->ats.enabled);
3496
3497                qdep  = dev_data->ats.qdep;
3498                iommu = amd_iommu_rlookup_table[dev_data->devid];
3499
3500                build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
3501                                      qdep, address, size);
3502
3503                ret = iommu_queue_command(iommu, &cmd);
3504                if (ret != 0)
3505                        goto out;
3506        }
3507
3508        /* Wait until all device TLBs are flushed */
3509        domain_flush_complete(domain);
3510
3511        ret = 0;
3512
3513out:
3514
3515        return ret;
3516}
3517
3518static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
3519                                  u64 address)
3520{
3521        INC_STATS_COUNTER(invalidate_iotlb);
3522
3523        return __flush_pasid(domain, pasid, address, false);
3524}
3525
3526int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
3527                         u64 address)
3528{
3529        struct protection_domain *domain = dom->priv;
3530        unsigned long flags;
3531        int ret;
3532
3533        spin_lock_irqsave(&domain->lock, flags);
3534        ret = __amd_iommu_flush_page(domain, pasid, address);
3535        spin_unlock_irqrestore(&domain->lock, flags);
3536
3537        return ret;
3538}
3539EXPORT_SYMBOL(amd_iommu_flush_page);
3540
3541static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
3542{
3543        INC_STATS_COUNTER(invalidate_iotlb_all);
3544
3545        return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
3546                             true);
3547}
3548
3549int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
3550{
3551        struct protection_domain *domain = dom->priv;
3552        unsigned long flags;
3553        int ret;
3554
3555        spin_lock_irqsave(&domain->lock, flags);
3556        ret = __amd_iommu_flush_tlb(domain, pasid);
3557        spin_unlock_irqrestore(&domain->lock, flags);
3558
3559        return ret;
3560}
3561EXPORT_SYMBOL(amd_iommu_flush_tlb);
3562
3563static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
3564{
3565        int index;
3566        u64 *pte;
3567
3568        while (true) {
3569
3570                index = (pasid >> (9 * level)) & 0x1ff;
3571                pte   = &root[index];
3572
3573                if (level == 0)
3574                        break;
3575
3576                if (!(*pte & GCR3_VALID)) {
3577                        if (!alloc)
3578                                return NULL;
3579
3580                        root = (void *)get_zeroed_page(GFP_ATOMIC);
3581                        if (root == NULL)
3582                                return NULL;
3583
3584                        *pte = __pa(root) | GCR3_VALID;
3585                }
3586
3587                root = __va(*pte & PAGE_MASK);
3588
3589                level -= 1;
3590        }
3591
3592        return pte;
3593}
3594
3595static int __set_gcr3(struct protection_domain *domain, int pasid,
3596                      unsigned long cr3)
3597{
3598        u64 *pte;
3599
3600        if (domain->mode != PAGE_MODE_NONE)
3601                return -EINVAL;
3602
3603        pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
3604        if (pte == NULL)
3605                return -ENOMEM;
3606
3607        *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
3608
3609        return __amd_iommu_flush_tlb(domain, pasid);
3610}
3611
3612static int __clear_gcr3(struct protection_domain *domain, int pasid)
3613{
3614        u64 *pte;
3615
3616        if (domain->mode != PAGE_MODE_NONE)
3617                return -EINVAL;
3618
3619        pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
3620        if (pte == NULL)
3621                return 0;
3622
3623        *pte = 0;
3624
3625        return __amd_iommu_flush_tlb(domain, pasid);
3626}
3627
3628int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
3629                              unsigned long cr3)
3630{
3631        struct protection_domain *domain = dom->priv;
3632        unsigned long flags;
3633        int ret;
3634
3635        spin_lock_irqsave(&domain->lock, flags);
3636        ret = __set_gcr3(domain, pasid, cr3);
3637        spin_unlock_irqrestore(&domain->lock, flags);
3638
3639        return ret;
3640}
3641EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
3642
3643int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
3644{
3645        struct protection_domain *domain = dom->priv;
3646        unsigned long flags;
3647        int ret;
3648
3649        spin_lock_irqsave(&domain->lock, flags);
3650        ret = __clear_gcr3(domain, pasid);
3651        spin_unlock_irqrestore(&domain->lock, flags);
3652
3653        return ret;
3654}
3655EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3656
3657int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
3658                           int status, int tag)
3659{
3660        struct iommu_dev_data *dev_data;
3661        struct amd_iommu *iommu;
3662        struct iommu_cmd cmd;
3663
3664        INC_STATS_COUNTER(complete_ppr);
3665
3666        dev_data = get_dev_data(&pdev->dev);
3667        iommu    = amd_iommu_rlookup_table[dev_data->devid];
3668
3669        build_complete_ppr(&cmd, dev_data->devid, pasid, status,
3670                           tag, dev_data->pri_tlp);
3671
3672        return iommu_queue_command(iommu, &cmd);
3673}
3674EXPORT_SYMBOL(amd_iommu_complete_ppr);
3675
3676struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
3677{
3678        struct protection_domain *domain;
3679
3680        domain = get_domain(&pdev->dev);
3681        if (IS_ERR(domain))
3682                return NULL;
3683
3684        /* Only return IOMMUv2 domains */
3685        if (!(domain->flags & PD_IOMMUV2_MASK))
3686                return NULL;
3687
3688        return domain->iommu_domain;
3689}
3690EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3691
3692void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
3693{
3694        struct iommu_dev_data *dev_data;
3695
3696        if (!amd_iommu_v2_supported())
3697                return;
3698
3699        dev_data = get_dev_data(&pdev->dev);
3700        dev_data->errata |= (1 << erratum);
3701}
3702EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3703
3704int amd_iommu_device_info(struct pci_dev *pdev,
3705                          struct amd_iommu_device_info *info)
3706{
3707        int max_pasids;
3708        int pos;
3709
3710        if (pdev == NULL || info == NULL)
3711                return -EINVAL;
3712
3713        if (!amd_iommu_v2_supported())
3714                return -EINVAL;
3715
3716        memset(info, 0, sizeof(*info));
3717
3718        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
3719        if (pos)
3720                info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
3721
3722        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
3723        if (pos)
3724                info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
3725
3726        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
3727        if (pos) {
3728                int features;
3729
3730                max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
3731                max_pasids = min(max_pasids, (1 << 20));
3732
3733                info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
3734                info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
3735
3736                features = pci_pasid_features(pdev);
3737                if (features & PCI_PASID_CAP_EXEC)
3738                        info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
3739                if (features & PCI_PASID_CAP_PRIV)
3740                        info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
3741        }
3742
3743        return 0;
3744}
3745EXPORT_SYMBOL(amd_iommu_device_info);
3746
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.