linux/drivers/iommu/dmar.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2006, Intel Corporation.
   3 *
   4 * This program is free software; you can redistribute it and/or modify it
   5 * under the terms and conditions of the GNU General Public License,
   6 * version 2, as published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope it will be useful, but WITHOUT
   9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 * more details.
  12 *
  13 * You should have received a copy of the GNU General Public License along with
  14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15 * Place - Suite 330, Boston, MA 02111-1307 USA.
  16 *
  17 * Copyright (C) 2006-2008 Intel Corporation
  18 * Author: Ashok Raj <ashok.raj@intel.com>
  19 * Author: Shaohua Li <shaohua.li@intel.com>
  20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21 *
  22 * This file implements early detection/parsing of Remapping Devices
  23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24 * tables.
  25 *
  26 * These routines are used by both DMA-remapping and Interrupt-remapping
  27 */
  28
  29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
  30
  31#include <linux/pci.h>
  32#include <linux/dmar.h>
  33#include <linux/iova.h>
  34#include <linux/intel-iommu.h>
  35#include <linux/timer.h>
  36#include <linux/irq.h>
  37#include <linux/interrupt.h>
  38#include <linux/tboot.h>
  39#include <linux/dmi.h>
  40#include <linux/slab.h>
  41#include <asm/irq_remapping.h>
  42#include <asm/iommu_table.h>
  43
  44#include "irq_remapping.h"
  45
  46/* No locks are needed as DMA remapping hardware unit
  47 * list is constructed at boot time and hotplug of
  48 * these units are not supported by the architecture.
  49 */
  50LIST_HEAD(dmar_drhd_units);
  51
  52struct acpi_table_header * __initdata dmar_tbl;
  53static acpi_size dmar_tbl_size;
  54
  55static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  56{
  57        /*
  58         * add INCLUDE_ALL at the tail, so scan the list will find it at
  59         * the very end.
  60         */
  61        if (drhd->include_all)
  62                list_add_tail(&drhd->list, &dmar_drhd_units);
  63        else
  64                list_add(&drhd->list, &dmar_drhd_units);
  65}
  66
  67static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  68                                           struct pci_dev **dev, u16 segment)
  69{
  70        struct pci_bus *bus;
  71        struct pci_dev *pdev = NULL;
  72        struct acpi_dmar_pci_path *path;
  73        int count;
  74
  75        bus = pci_find_bus(segment, scope->bus);
  76        path = (struct acpi_dmar_pci_path *)(scope + 1);
  77        count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  78                / sizeof(struct acpi_dmar_pci_path);
  79
  80        while (count) {
  81                if (pdev)
  82                        pci_dev_put(pdev);
  83                /*
  84                 * Some BIOSes list non-exist devices in DMAR table, just
  85                 * ignore it
  86                 */
  87                if (!bus) {
  88                        pr_warn("Device scope bus [%d] not found\n", scope->bus);
  89                        break;
  90                }
  91                pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  92                if (!pdev) {
  93                        /* warning will be printed below */
  94                        break;
  95                }
  96                path ++;
  97                count --;
  98                bus = pdev->subordinate;
  99        }
 100        if (!pdev) {
 101                pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
 102                        segment, scope->bus, path->dev, path->fn);
 103                *dev = NULL;
 104                return 0;
 105        }
 106        if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 107                        pdev->subordinate) || (scope->entry_type == \
 108                        ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 109                pci_dev_put(pdev);
 110                pr_warn("Device scope type does not match for %s\n",
 111                        pci_name(pdev));
 112                return -EINVAL;
 113        }
 114        *dev = pdev;
 115        return 0;
 116}
 117
 118int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 119                                struct pci_dev ***devices, u16 segment)
 120{
 121        struct acpi_dmar_device_scope *scope;
 122        void * tmp = start;
 123        int index;
 124        int ret;
 125
 126        *cnt = 0;
 127        while (start < end) {
 128                scope = start;
 129                if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 130                    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 131                        (*cnt)++;
 132                else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
 133                        pr_warn("Unsupported device scope\n");
 134                }
 135                start += scope->length;
 136        }
 137        if (*cnt == 0)
 138                return 0;
 139
 140        *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 141        if (!*devices)
 142                return -ENOMEM;
 143
 144        start = tmp;
 145        index = 0;
 146        while (start < end) {
 147                scope = start;
 148                if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 149                    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 150                        ret = dmar_parse_one_dev_scope(scope,
 151                                &(*devices)[index], segment);
 152                        if (ret) {
 153                                kfree(*devices);
 154                                return ret;
 155                        }
 156                        index ++;
 157                }
 158                start += scope->length;
 159        }
 160
 161        return 0;
 162}
 163
 164/**
 165 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 166 * structure which uniquely represent one DMA remapping hardware unit
 167 * present in the platform
 168 */
 169static int __init
 170dmar_parse_one_drhd(struct acpi_dmar_header *header)
 171{
 172        struct acpi_dmar_hardware_unit *drhd;
 173        struct dmar_drhd_unit *dmaru;
 174        int ret = 0;
 175
 176        drhd = (struct acpi_dmar_hardware_unit *)header;
 177        dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 178        if (!dmaru)
 179                return -ENOMEM;
 180
 181        dmaru->hdr = header;
 182        dmaru->reg_base_addr = drhd->address;
 183        dmaru->segment = drhd->segment;
 184        dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 185
 186        ret = alloc_iommu(dmaru);
 187        if (ret) {
 188                kfree(dmaru);
 189                return ret;
 190        }
 191        dmar_register_drhd_unit(dmaru);
 192        return 0;
 193}
 194
 195static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 196{
 197        struct acpi_dmar_hardware_unit *drhd;
 198        int ret = 0;
 199
 200        drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 201
 202        if (dmaru->include_all)
 203                return 0;
 204
 205        ret = dmar_parse_dev_scope((void *)(drhd + 1),
 206                                ((void *)drhd) + drhd->header.length,
 207                                &dmaru->devices_cnt, &dmaru->devices,
 208                                drhd->segment);
 209        if (ret) {
 210                list_del(&dmaru->list);
 211                kfree(dmaru);
 212        }
 213        return ret;
 214}
 215
 216#ifdef CONFIG_ACPI_NUMA
 217static int __init
 218dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 219{
 220        struct acpi_dmar_rhsa *rhsa;
 221        struct dmar_drhd_unit *drhd;
 222
 223        rhsa = (struct acpi_dmar_rhsa *)header;
 224        for_each_drhd_unit(drhd) {
 225                if (drhd->reg_base_addr == rhsa->base_address) {
 226                        int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 227
 228                        if (!node_online(node))
 229                                node = -1;
 230                        drhd->iommu->node = node;
 231                        return 0;
 232                }
 233        }
 234        WARN_TAINT(
 235                1, TAINT_FIRMWARE_WORKAROUND,
 236                "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 237                "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 238                drhd->reg_base_addr,
 239                dmi_get_system_info(DMI_BIOS_VENDOR),
 240                dmi_get_system_info(DMI_BIOS_VERSION),
 241                dmi_get_system_info(DMI_PRODUCT_VERSION));
 242
 243        return 0;
 244}
 245#endif
 246
 247static void __init
 248dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 249{
 250        struct acpi_dmar_hardware_unit *drhd;
 251        struct acpi_dmar_reserved_memory *rmrr;
 252        struct acpi_dmar_atsr *atsr;
 253        struct acpi_dmar_rhsa *rhsa;
 254
 255        switch (header->type) {
 256        case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 257                drhd = container_of(header, struct acpi_dmar_hardware_unit,
 258                                    header);
 259                pr_info("DRHD base: %#016Lx flags: %#x\n",
 260                        (unsigned long long)drhd->address, drhd->flags);
 261                break;
 262        case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 263                rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 264                                    header);
 265                pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 266                        (unsigned long long)rmrr->base_address,
 267                        (unsigned long long)rmrr->end_address);
 268                break;
 269        case ACPI_DMAR_TYPE_ATSR:
 270                atsr = container_of(header, struct acpi_dmar_atsr, header);
 271                pr_info("ATSR flags: %#x\n", atsr->flags);
 272                break;
 273        case ACPI_DMAR_HARDWARE_AFFINITY:
 274                rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 275                pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 276                       (unsigned long long)rhsa->base_address,
 277                       rhsa->proximity_domain);
 278                break;
 279        }
 280}
 281
 282/**
 283 * dmar_table_detect - checks to see if the platform supports DMAR devices
 284 */
 285static int __init dmar_table_detect(void)
 286{
 287        acpi_status status = AE_OK;
 288
 289        /* if we could find DMAR table, then there are DMAR devices */
 290        status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 291                                (struct acpi_table_header **)&dmar_tbl,
 292                                &dmar_tbl_size);
 293
 294        if (ACPI_SUCCESS(status) && !dmar_tbl) {
 295                pr_warn("Unable to map DMAR\n");
 296                status = AE_NOT_FOUND;
 297        }
 298
 299        return (ACPI_SUCCESS(status) ? 1 : 0);
 300}
 301
 302/**
 303 * parse_dmar_table - parses the DMA reporting table
 304 */
 305static int __init
 306parse_dmar_table(void)
 307{
 308        struct acpi_table_dmar *dmar;
 309        struct acpi_dmar_header *entry_header;
 310        int ret = 0;
 311
 312        /*
 313         * Do it again, earlier dmar_tbl mapping could be mapped with
 314         * fixed map.
 315         */
 316        dmar_table_detect();
 317
 318        /*
 319         * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 320         * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 321         */
 322        dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 323
 324        dmar = (struct acpi_table_dmar *)dmar_tbl;
 325        if (!dmar)
 326                return -ENODEV;
 327
 328        if (dmar->width < PAGE_SHIFT - 1) {
 329                pr_warn("Invalid DMAR haw\n");
 330                return -EINVAL;
 331        }
 332
 333        pr_info("Host address width %d\n", dmar->width + 1);
 334
 335        entry_header = (struct acpi_dmar_header *)(dmar + 1);
 336        while (((unsigned long)entry_header) <
 337                        (((unsigned long)dmar) + dmar_tbl->length)) {
 338                /* Avoid looping forever on bad ACPI tables */
 339                if (entry_header->length == 0) {
 340                        pr_warn("Invalid 0-length structure\n");
 341                        ret = -EINVAL;
 342                        break;
 343                }
 344
 345                dmar_table_print_dmar_entry(entry_header);
 346
 347                switch (entry_header->type) {
 348                case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 349                        ret = dmar_parse_one_drhd(entry_header);
 350                        break;
 351                case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 352                        ret = dmar_parse_one_rmrr(entry_header);
 353                        break;
 354                case ACPI_DMAR_TYPE_ATSR:
 355                        ret = dmar_parse_one_atsr(entry_header);
 356                        break;
 357                case ACPI_DMAR_HARDWARE_AFFINITY:
 358#ifdef CONFIG_ACPI_NUMA
 359                        ret = dmar_parse_one_rhsa(entry_header);
 360#endif
 361                        break;
 362                default:
 363                        pr_warn("Unknown DMAR structure type %d\n",
 364                                entry_header->type);
 365                        ret = 0; /* for forward compatibility */
 366                        break;
 367                }
 368                if (ret)
 369                        break;
 370
 371                entry_header = ((void *)entry_header + entry_header->length);
 372        }
 373        return ret;
 374}
 375
 376static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 377                          struct pci_dev *dev)
 378{
 379        int index;
 380
 381        while (dev) {
 382                for (index = 0; index < cnt; index++)
 383                        if (dev == devices[index])
 384                                return 1;
 385
 386                /* Check our parent */
 387                dev = dev->bus->self;
 388        }
 389
 390        return 0;
 391}
 392
 393struct dmar_drhd_unit *
 394dmar_find_matched_drhd_unit(struct pci_dev *dev)
 395{
 396        struct dmar_drhd_unit *dmaru = NULL;
 397        struct acpi_dmar_hardware_unit *drhd;
 398
 399        dev = pci_physfn(dev);
 400
 401        list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 402                drhd = container_of(dmaru->hdr,
 403                                    struct acpi_dmar_hardware_unit,
 404                                    header);
 405
 406                if (dmaru->include_all &&
 407                    drhd->segment == pci_domain_nr(dev->bus))
 408                        return dmaru;
 409
 410                if (dmar_pci_device_match(dmaru->devices,
 411                                          dmaru->devices_cnt, dev))
 412                        return dmaru;
 413        }
 414
 415        return NULL;
 416}
 417
 418int __init dmar_dev_scope_init(void)
 419{
 420        static int dmar_dev_scope_initialized;
 421        struct dmar_drhd_unit *drhd, *drhd_n;
 422        int ret = -ENODEV;
 423
 424        if (dmar_dev_scope_initialized)
 425                return dmar_dev_scope_initialized;
 426
 427        if (list_empty(&dmar_drhd_units))
 428                goto fail;
 429
 430        list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 431                ret = dmar_parse_dev(drhd);
 432                if (ret)
 433                        goto fail;
 434        }
 435
 436        ret = dmar_parse_rmrr_atsr_dev();
 437        if (ret)
 438                goto fail;
 439
 440        dmar_dev_scope_initialized = 1;
 441        return 0;
 442
 443fail:
 444        dmar_dev_scope_initialized = ret;
 445        return ret;
 446}
 447
 448
 449int __init dmar_table_init(void)
 450{
 451        static int dmar_table_initialized;
 452        int ret;
 453
 454        if (dmar_table_initialized)
 455                return 0;
 456
 457        dmar_table_initialized = 1;
 458
 459        ret = parse_dmar_table();
 460        if (ret) {
 461                if (ret != -ENODEV)
 462                        pr_info("parse DMAR table failure.\n");
 463                return ret;
 464        }
 465
 466        if (list_empty(&dmar_drhd_units)) {
 467                pr_info("No DMAR devices found\n");
 468                return -ENODEV;
 469        }
 470
 471        return 0;
 472}
 473
 474static void warn_invalid_dmar(u64 addr, const char *message)
 475{
 476        WARN_TAINT_ONCE(
 477                1, TAINT_FIRMWARE_WORKAROUND,
 478                "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 479                "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 480                addr, message,
 481                dmi_get_system_info(DMI_BIOS_VENDOR),
 482                dmi_get_system_info(DMI_BIOS_VERSION),
 483                dmi_get_system_info(DMI_PRODUCT_VERSION));
 484}
 485
 486int __init check_zero_address(void)
 487{
 488        struct acpi_table_dmar *dmar;
 489        struct acpi_dmar_header *entry_header;
 490        struct acpi_dmar_hardware_unit *drhd;
 491
 492        dmar = (struct acpi_table_dmar *)dmar_tbl;
 493        entry_header = (struct acpi_dmar_header *)(dmar + 1);
 494
 495        while (((unsigned long)entry_header) <
 496                        (((unsigned long)dmar) + dmar_tbl->length)) {
 497                /* Avoid looping forever on bad ACPI tables */
 498                if (entry_header->length == 0) {
 499                        pr_warn("Invalid 0-length structure\n");
 500                        return 0;
 501                }
 502
 503                if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 504                        void __iomem *addr;
 505                        u64 cap, ecap;
 506
 507                        drhd = (void *)entry_header;
 508                        if (!drhd->address) {
 509                                warn_invalid_dmar(0, "");
 510                                goto failed;
 511                        }
 512
 513                        addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 514                        if (!addr ) {
 515                                printk("IOMMU: can't validate: %llx\n", drhd->address);
 516                                goto failed;
 517                        }
 518                        cap = dmar_readq(addr + DMAR_CAP_REG);
 519                        ecap = dmar_readq(addr + DMAR_ECAP_REG);
 520                        early_iounmap(addr, VTD_PAGE_SIZE);
 521                        if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 522                                warn_invalid_dmar(drhd->address,
 523                                                  " returns all ones");
 524                                goto failed;
 525                        }
 526                }
 527
 528                entry_header = ((void *)entry_header + entry_header->length);
 529        }
 530        return 1;
 531
 532failed:
 533        return 0;
 534}
 535
 536int __init detect_intel_iommu(void)
 537{
 538        int ret;
 539
 540        ret = dmar_table_detect();
 541        if (ret)
 542                ret = check_zero_address();
 543        {
 544                struct acpi_table_dmar *dmar;
 545
 546                dmar = (struct acpi_table_dmar *) dmar_tbl;
 547
 548                if (ret && irq_remapping_enabled && cpu_has_x2apic &&
 549                    dmar->flags & 0x1)
 550                        pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
 551
 552                if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 553                        iommu_detected = 1;
 554                        /* Make sure ACS will be enabled */
 555                        pci_request_acs();
 556                }
 557
 558#ifdef CONFIG_X86
 559                if (ret)
 560                        x86_init.iommu.iommu_init = intel_iommu_init;
 561#endif
 562        }
 563        early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 564        dmar_tbl = NULL;
 565
 566        return ret ? 1 : -ENODEV;
 567}
 568
 569
 570static void unmap_iommu(struct intel_iommu *iommu)
 571{
 572        iounmap(iommu->reg);
 573        release_mem_region(iommu->reg_phys, iommu->reg_size);
 574}
 575
 576/**
 577 * map_iommu: map the iommu's registers
 578 * @iommu: the iommu to map
 579 * @phys_addr: the physical address of the base resgister
 580 *
 581 * Memory map the iommu's registers.  Start w/ a single page, and
 582 * possibly expand if that turns out to be insufficent.
 583 */
 584static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 585{
 586        int map_size, err=0;
 587
 588        iommu->reg_phys = phys_addr;
 589        iommu->reg_size = VTD_PAGE_SIZE;
 590
 591        if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 592                pr_err("IOMMU: can't reserve memory\n");
 593                err = -EBUSY;
 594                goto out;
 595        }
 596
 597        iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 598        if (!iommu->reg) {
 599                pr_err("IOMMU: can't map the region\n");
 600                err = -ENOMEM;
 601                goto release;
 602        }
 603
 604        iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 605        iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 606
 607        if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 608                err = -EINVAL;
 609                warn_invalid_dmar(phys_addr, " returns all ones");
 610                goto unmap;
 611        }
 612
 613        /* the registers might be more than one page */
 614        map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 615                         cap_max_fault_reg_offset(iommu->cap));
 616        map_size = VTD_PAGE_ALIGN(map_size);
 617        if (map_size > iommu->reg_size) {
 618                iounmap(iommu->reg);
 619                release_mem_region(iommu->reg_phys, iommu->reg_size);
 620                iommu->reg_size = map_size;
 621                if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 622                                        iommu->name)) {
 623                        pr_err("IOMMU: can't reserve memory\n");
 624                        err = -EBUSY;
 625                        goto out;
 626                }
 627                iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 628                if (!iommu->reg) {
 629                        pr_err("IOMMU: can't map the region\n");
 630                        err = -ENOMEM;
 631                        goto release;
 632                }
 633        }
 634        err = 0;
 635        goto out;
 636
 637unmap:
 638        iounmap(iommu->reg);
 639release:
 640        release_mem_region(iommu->reg_phys, iommu->reg_size);
 641out:
 642        return err;
 643}
 644
 645int alloc_iommu(struct dmar_drhd_unit *drhd)
 646{
 647        struct intel_iommu *iommu;
 648        u32 ver;
 649        static int iommu_allocated = 0;
 650        int agaw = 0;
 651        int msagaw = 0;
 652        int err;
 653
 654        if (!drhd->reg_base_addr) {
 655                warn_invalid_dmar(0, "");
 656                return -EINVAL;
 657        }
 658
 659        iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 660        if (!iommu)
 661                return -ENOMEM;
 662
 663        iommu->seq_id = iommu_allocated++;
 664        sprintf (iommu->name, "dmar%d", iommu->seq_id);
 665
 666        err = map_iommu(iommu, drhd->reg_base_addr);
 667        if (err) {
 668                pr_err("IOMMU: failed to map %s\n", iommu->name);
 669                goto error;
 670        }
 671
 672        err = -EINVAL;
 673        agaw = iommu_calculate_agaw(iommu);
 674        if (agaw < 0) {
 675                pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
 676                        iommu->seq_id);
 677                goto err_unmap;
 678        }
 679        msagaw = iommu_calculate_max_sagaw(iommu);
 680        if (msagaw < 0) {
 681                pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 682                        iommu->seq_id);
 683                goto err_unmap;
 684        }
 685        iommu->agaw = agaw;
 686        iommu->msagaw = msagaw;
 687
 688        iommu->node = -1;
 689
 690        ver = readl(iommu->reg + DMAR_VER_REG);
 691        pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 692                iommu->seq_id,
 693                (unsigned long long)drhd->reg_base_addr,
 694                DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 695                (unsigned long long)iommu->cap,
 696                (unsigned long long)iommu->ecap);
 697
 698        raw_spin_lock_init(&iommu->register_lock);
 699
 700        drhd->iommu = iommu;
 701        return 0;
 702
 703 err_unmap:
 704        unmap_iommu(iommu);
 705 error:
 706        kfree(iommu);
 707        return err;
 708}
 709
 710void free_iommu(struct intel_iommu *iommu)
 711{
 712        if (!iommu)
 713                return;
 714
 715        free_dmar_iommu(iommu);
 716
 717        if (iommu->reg)
 718                unmap_iommu(iommu);
 719
 720        kfree(iommu);
 721}
 722
 723/*
 724 * Reclaim all the submitted descriptors which have completed its work.
 725 */
 726static inline void reclaim_free_desc(struct q_inval *qi)
 727{
 728        while (qi->desc_status[qi->free_tail] == QI_DONE ||
 729               qi->desc_status[qi->free_tail] == QI_ABORT) {
 730                qi->desc_status[qi->free_tail] = QI_FREE;
 731                qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 732                qi->free_cnt++;
 733        }
 734}
 735
 736static int qi_check_fault(struct intel_iommu *iommu, int index)
 737{
 738        u32 fault;
 739        int head, tail;
 740        struct q_inval *qi = iommu->qi;
 741        int wait_index = (index + 1) % QI_LENGTH;
 742
 743        if (qi->desc_status[wait_index] == QI_ABORT)
 744                return -EAGAIN;
 745
 746        fault = readl(iommu->reg + DMAR_FSTS_REG);
 747
 748        /*
 749         * If IQE happens, the head points to the descriptor associated
 750         * with the error. No new descriptors are fetched until the IQE
 751         * is cleared.
 752         */
 753        if (fault & DMA_FSTS_IQE) {
 754                head = readl(iommu->reg + DMAR_IQH_REG);
 755                if ((head >> DMAR_IQ_SHIFT) == index) {
 756                        pr_err("VT-d detected invalid descriptor: "
 757                                "low=%llx, high=%llx\n",
 758                                (unsigned long long)qi->desc[index].low,
 759                                (unsigned long long)qi->desc[index].high);
 760                        memcpy(&qi->desc[index], &qi->desc[wait_index],
 761                                        sizeof(struct qi_desc));
 762                        __iommu_flush_cache(iommu, &qi->desc[index],
 763                                        sizeof(struct qi_desc));
 764                        writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 765                        return -EINVAL;
 766                }
 767        }
 768
 769        /*
 770         * If ITE happens, all pending wait_desc commands are aborted.
 771         * No new descriptors are fetched until the ITE is cleared.
 772         */
 773        if (fault & DMA_FSTS_ITE) {
 774                head = readl(iommu->reg + DMAR_IQH_REG);
 775                head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 776                head |= 1;
 777                tail = readl(iommu->reg + DMAR_IQT_REG);
 778                tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 779
 780                writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 781
 782                do {
 783                        if (qi->desc_status[head] == QI_IN_USE)
 784                                qi->desc_status[head] = QI_ABORT;
 785                        head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 786                } while (head != tail);
 787
 788                if (qi->desc_status[wait_index] == QI_ABORT)
 789                        return -EAGAIN;
 790        }
 791
 792        if (fault & DMA_FSTS_ICE)
 793                writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 794
 795        return 0;
 796}
 797
 798/*
 799 * Submit the queued invalidation descriptor to the remapping
 800 * hardware unit and wait for its completion.
 801 */
 802int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 803{
 804        int rc;
 805        struct q_inval *qi = iommu->qi;
 806        struct qi_desc *hw, wait_desc;
 807        int wait_index, index;
 808        unsigned long flags;
 809
 810        if (!qi)
 811                return 0;
 812
 813        hw = qi->desc;
 814
 815restart:
 816        rc = 0;
 817
 818        raw_spin_lock_irqsave(&qi->q_lock, flags);
 819        while (qi->free_cnt < 3) {
 820                raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 821                cpu_relax();
 822                raw_spin_lock_irqsave(&qi->q_lock, flags);
 823        }
 824
 825        index = qi->free_head;
 826        wait_index = (index + 1) % QI_LENGTH;
 827
 828        qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 829
 830        hw[index] = *desc;
 831
 832        wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 833                        QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 834        wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 835
 836        hw[wait_index] = wait_desc;
 837
 838        __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 839        __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 840
 841        qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 842        qi->free_cnt -= 2;
 843
 844        /*
 845         * update the HW tail register indicating the presence of
 846         * new descriptors.
 847         */
 848        writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 849
 850        while (qi->desc_status[wait_index] != QI_DONE) {
 851                /*
 852                 * We will leave the interrupts disabled, to prevent interrupt
 853                 * context to queue another cmd while a cmd is already submitted
 854                 * and waiting for completion on this cpu. This is to avoid
 855                 * a deadlock where the interrupt context can wait indefinitely
 856                 * for free slots in the queue.
 857                 */
 858                rc = qi_check_fault(iommu, index);
 859                if (rc)
 860                        break;
 861
 862                raw_spin_unlock(&qi->q_lock);
 863                cpu_relax();
 864                raw_spin_lock(&qi->q_lock);
 865        }
 866
 867        qi->desc_status[index] = QI_DONE;
 868
 869        reclaim_free_desc(qi);
 870        raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 871
 872        if (rc == -EAGAIN)
 873                goto restart;
 874
 875        return rc;
 876}
 877
 878/*
 879 * Flush the global interrupt entry cache.
 880 */
 881void qi_global_iec(struct intel_iommu *iommu)
 882{
 883        struct qi_desc desc;
 884
 885        desc.low = QI_IEC_TYPE;
 886        desc.high = 0;
 887
 888        /* should never fail */
 889        qi_submit_sync(&desc, iommu);
 890}
 891
 892void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 893                      u64 type)
 894{
 895        struct qi_desc desc;
 896
 897        desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 898                        | QI_CC_GRAN(type) | QI_CC_TYPE;
 899        desc.high = 0;
 900
 901        qi_submit_sync(&desc, iommu);
 902}
 903
 904void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 905                    unsigned int size_order, u64 type)
 906{
 907        u8 dw = 0, dr = 0;
 908
 909        struct qi_desc desc;
 910        int ih = 0;
 911
 912        if (cap_write_drain(iommu->cap))
 913                dw = 1;
 914
 915        if (cap_read_drain(iommu->cap))
 916                dr = 1;
 917
 918        desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 919                | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 920        desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 921                | QI_IOTLB_AM(size_order);
 922
 923        qi_submit_sync(&desc, iommu);
 924}
 925
 926void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 927                        u64 addr, unsigned mask)
 928{
 929        struct qi_desc desc;
 930
 931        if (mask) {
 932                BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 933                addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 934                desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 935        } else
 936                desc.high = QI_DEV_IOTLB_ADDR(addr);
 937
 938        if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 939                qdep = 0;
 940
 941        desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 942                   QI_DIOTLB_TYPE;
 943
 944        qi_submit_sync(&desc, iommu);
 945}
 946
 947/*
 948 * Disable Queued Invalidation interface.
 949 */
 950void dmar_disable_qi(struct intel_iommu *iommu)
 951{
 952        unsigned long flags;
 953        u32 sts;
 954        cycles_t start_time = get_cycles();
 955
 956        if (!ecap_qis(iommu->ecap))
 957                return;
 958
 959        raw_spin_lock_irqsave(&iommu->register_lock, flags);
 960
 961        sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 962        if (!(sts & DMA_GSTS_QIES))
 963                goto end;
 964
 965        /*
 966         * Give a chance to HW to complete the pending invalidation requests.
 967         */
 968        while ((readl(iommu->reg + DMAR_IQT_REG) !=
 969                readl(iommu->reg + DMAR_IQH_REG)) &&
 970                (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 971                cpu_relax();
 972
 973        iommu->gcmd &= ~DMA_GCMD_QIE;
 974        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 975
 976        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 977                      !(sts & DMA_GSTS_QIES), sts);
 978end:
 979        raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
 980}
 981
 982/*
 983 * Enable queued invalidation.
 984 */
 985static void __dmar_enable_qi(struct intel_iommu *iommu)
 986{
 987        u32 sts;
 988        unsigned long flags;
 989        struct q_inval *qi = iommu->qi;
 990
 991        qi->free_head = qi->free_tail = 0;
 992        qi->free_cnt = QI_LENGTH;
 993
 994        raw_spin_lock_irqsave(&iommu->register_lock, flags);
 995
 996        /* write zero to the tail reg */
 997        writel(0, iommu->reg + DMAR_IQT_REG);
 998
 999        dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1000
1001        iommu->gcmd |= DMA_GCMD_QIE;
1002        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1003
1004        /* Make sure hardware complete it */
1005        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1006
1007        raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1008}
1009
1010/*
1011 * Enable Queued Invalidation interface. This is a must to support
1012 * interrupt-remapping. Also used by DMA-remapping, which replaces
1013 * register based IOTLB invalidation.
1014 */
1015int dmar_enable_qi(struct intel_iommu *iommu)
1016{
1017        struct q_inval *qi;
1018        struct page *desc_page;
1019
1020        if (!ecap_qis(iommu->ecap))
1021                return -ENOENT;
1022
1023        /*
1024         * queued invalidation is already setup and enabled.
1025         */
1026        if (iommu->qi)
1027                return 0;
1028
1029        iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1030        if (!iommu->qi)
1031                return -ENOMEM;
1032
1033        qi = iommu->qi;
1034
1035
1036        desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1037        if (!desc_page) {
1038                kfree(qi);
1039                iommu->qi = 0;
1040                return -ENOMEM;
1041        }
1042
1043        qi->desc = page_address(desc_page);
1044
1045        qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1046        if (!qi->desc_status) {
1047                free_page((unsigned long) qi->desc);
1048                kfree(qi);
1049                iommu->qi = 0;
1050                return -ENOMEM;
1051        }
1052
1053        qi->free_head = qi->free_tail = 0;
1054        qi->free_cnt = QI_LENGTH;
1055
1056        raw_spin_lock_init(&qi->q_lock);
1057
1058        __dmar_enable_qi(iommu);
1059
1060        return 0;
1061}
1062
1063/* iommu interrupt handling. Most stuff are MSI-like. */
1064
1065enum faulttype {
1066        DMA_REMAP,
1067        INTR_REMAP,
1068        UNKNOWN,
1069};
1070
1071static const char *dma_remap_fault_reasons[] =
1072{
1073        "Software",
1074        "Present bit in root entry is clear",
1075        "Present bit in context entry is clear",
1076        "Invalid context entry",
1077        "Access beyond MGAW",
1078        "PTE Write access is not set",
1079        "PTE Read access is not set",
1080        "Next page table ptr is invalid",
1081        "Root table address invalid",
1082        "Context table ptr is invalid",
1083        "non-zero reserved fields in RTP",
1084        "non-zero reserved fields in CTP",
1085        "non-zero reserved fields in PTE",
1086        "PCE for translation request specifies blocking",
1087};
1088
1089static const char *irq_remap_fault_reasons[] =
1090{
1091        "Detected reserved fields in the decoded interrupt-remapped request",
1092        "Interrupt index exceeded the interrupt-remapping table size",
1093        "Present field in the IRTE entry is clear",
1094        "Error accessing interrupt-remapping table pointed by IRTA_REG",
1095        "Detected reserved fields in the IRTE entry",
1096        "Blocked a compatibility format interrupt request",
1097        "Blocked an interrupt request due to source-id verification failure",
1098};
1099
1100#define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1101
1102const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1103{
1104        if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1105                                        ARRAY_SIZE(irq_remap_fault_reasons))) {
1106                *fault_type = INTR_REMAP;
1107                return irq_remap_fault_reasons[fault_reason - 0x20];
1108        } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1109                *fault_type = DMA_REMAP;
1110                return dma_remap_fault_reasons[fault_reason];
1111        } else {
1112                *fault_type = UNKNOWN;
1113                return "Unknown";
1114        }
1115}
1116
1117void dmar_msi_unmask(struct irq_data *data)
1118{
1119        struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1120        unsigned long flag;
1121
1122        /* unmask it */
1123        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1124        writel(0, iommu->reg + DMAR_FECTL_REG);
1125        /* Read a reg to force flush the post write */
1126        readl(iommu->reg + DMAR_FECTL_REG);
1127        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1128}
1129
1130void dmar_msi_mask(struct irq_data *data)
1131{
1132        unsigned long flag;
1133        struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1134
1135        /* mask it */
1136        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1137        writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1138        /* Read a reg to force flush the post write */
1139        readl(iommu->reg + DMAR_FECTL_REG);
1140        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1141}
1142
1143void dmar_msi_write(int irq, struct msi_msg *msg)
1144{
1145        struct intel_iommu *iommu = irq_get_handler_data(irq);
1146        unsigned long flag;
1147
1148        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1149        writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1150        writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1151        writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1152        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1153}
1154
1155void dmar_msi_read(int irq, struct msi_msg *msg)
1156{
1157        struct intel_iommu *iommu = irq_get_handler_data(irq);
1158        unsigned long flag;
1159
1160        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1161        msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1162        msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1163        msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1164        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1165}
1166
1167static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1168                u8 fault_reason, u16 source_id, unsigned long long addr)
1169{
1170        const char *reason;
1171        int fault_type;
1172
1173        reason = dmar_get_fault_reason(fault_reason, &fault_type);
1174
1175        if (fault_type == INTR_REMAP)
1176                pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1177                       "fault index %llx\n"
1178                        "INTR-REMAP:[fault reason %02d] %s\n",
1179                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1180                        PCI_FUNC(source_id & 0xFF), addr >> 48,
1181                        fault_reason, reason);
1182        else
1183                pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1184                       "fault addr %llx \n"
1185                       "DMAR:[fault reason %02d] %s\n",
1186                       (type ? "DMA Read" : "DMA Write"),
1187                       (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1188                       PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1189        return 0;
1190}
1191
1192#define PRIMARY_FAULT_REG_LEN (16)
1193irqreturn_t dmar_fault(int irq, void *dev_id)
1194{
1195        struct intel_iommu *iommu = dev_id;
1196        int reg, fault_index;
1197        u32 fault_status;
1198        unsigned long flag;
1199
1200        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1201        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1202        if (fault_status)
1203                pr_err("DRHD: handling fault status reg %x\n", fault_status);
1204
1205        /* TBD: ignore advanced fault log currently */
1206        if (!(fault_status & DMA_FSTS_PPF))
1207                goto clear_rest;
1208
1209        fault_index = dma_fsts_fault_record_index(fault_status);
1210        reg = cap_fault_reg_offset(iommu->cap);
1211        while (1) {
1212                u8 fault_reason;
1213                u16 source_id;
1214                u64 guest_addr;
1215                int type;
1216                u32 data;
1217
1218                /* highest 32 bits */
1219                data = readl(iommu->reg + reg +
1220                                fault_index * PRIMARY_FAULT_REG_LEN + 12);
1221                if (!(data & DMA_FRCD_F))
1222                        break;
1223
1224                fault_reason = dma_frcd_fault_reason(data);
1225                type = dma_frcd_type(data);
1226
1227                data = readl(iommu->reg + reg +
1228                                fault_index * PRIMARY_FAULT_REG_LEN + 8);
1229                source_id = dma_frcd_source_id(data);
1230
1231                guest_addr = dmar_readq(iommu->reg + reg +
1232                                fault_index * PRIMARY_FAULT_REG_LEN);
1233                guest_addr = dma_frcd_page_addr(guest_addr);
1234                /* clear the fault */
1235                writel(DMA_FRCD_F, iommu->reg + reg +
1236                        fault_index * PRIMARY_FAULT_REG_LEN + 12);
1237
1238                raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1239
1240                dmar_fault_do_one(iommu, type, fault_reason,
1241                                source_id, guest_addr);
1242
1243                fault_index++;
1244                if (fault_index >= cap_num_fault_regs(iommu->cap))
1245                        fault_index = 0;
1246                raw_spin_lock_irqsave(&iommu->register_lock, flag);
1247        }
1248clear_rest:
1249        /* clear all the other faults */
1250        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1251        writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1252
1253        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1254        return IRQ_HANDLED;
1255}
1256
1257int dmar_set_interrupt(struct intel_iommu *iommu)
1258{
1259        int irq, ret;
1260
1261        /*
1262         * Check if the fault interrupt is already initialized.
1263         */
1264        if (iommu->irq)
1265                return 0;
1266
1267        irq = create_irq();
1268        if (!irq) {
1269                pr_err("IOMMU: no free vectors\n");
1270                return -EINVAL;
1271        }
1272
1273        irq_set_handler_data(irq, iommu);
1274        iommu->irq = irq;
1275
1276        ret = arch_setup_dmar_msi(irq);
1277        if (ret) {
1278                irq_set_handler_data(irq, NULL);
1279                iommu->irq = 0;
1280                destroy_irq(irq);
1281                return ret;
1282        }
1283
1284        ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1285        if (ret)
1286                pr_err("IOMMU: can't request irq\n");
1287        return ret;
1288}
1289
1290int __init enable_drhd_fault_handling(void)
1291{
1292        struct dmar_drhd_unit *drhd;
1293
1294        /*
1295         * Enable fault control interrupt.
1296         */
1297        for_each_drhd_unit(drhd) {
1298                int ret;
1299                struct intel_iommu *iommu = drhd->iommu;
1300                ret = dmar_set_interrupt(iommu);
1301
1302                if (ret) {
1303                        pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1304                               (unsigned long long)drhd->reg_base_addr, ret);
1305                        return -1;
1306                }
1307
1308                /*
1309                 * Clear any previous faults.
1310                 */
1311                dmar_fault(iommu->irq, iommu);
1312        }
1313
1314        return 0;
1315}
1316
1317/*
1318 * Re-enable Queued Invalidation interface.
1319 */
1320int dmar_reenable_qi(struct intel_iommu *iommu)
1321{
1322        if (!ecap_qis(iommu->ecap))
1323                return -ENOENT;
1324
1325        if (!iommu->qi)
1326                return -ENOENT;
1327
1328        /*
1329         * First disable queued invalidation.
1330         */
1331        dmar_disable_qi(iommu);
1332        /*
1333         * Then enable queued invalidation again. Since there is no pending
1334         * invalidation requests now, it's safe to re-enable queued
1335         * invalidation.
1336         */
1337        __dmar_enable_qi(iommu);
1338
1339        return 0;
1340}
1341
1342/*
1343 * Check interrupt remapping support in DMAR table description.
1344 */
1345int __init dmar_ir_support(void)
1346{
1347        struct acpi_table_dmar *dmar;
1348        dmar = (struct acpi_table_dmar *)dmar_tbl;
1349        if (!dmar)
1350                return 0;
1351        return dmar->flags & 0x1;
1352}
1353IOMMU_INIT_POST(detect_intel_iommu);
1354
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.