linux/drivers/iommu/dmar.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2006, Intel Corporation.
   3 *
   4 * This program is free software; you can redistribute it and/or modify it
   5 * under the terms and conditions of the GNU General Public License,
   6 * version 2, as published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope it will be useful, but WITHOUT
   9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 * more details.
  12 *
  13 * You should have received a copy of the GNU General Public License along with
  14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15 * Place - Suite 330, Boston, MA 02111-1307 USA.
  16 *
  17 * Copyright (C) 2006-2008 Intel Corporation
  18 * Author: Ashok Raj <ashok.raj@intel.com>
  19 * Author: Shaohua Li <shaohua.li@intel.com>
  20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21 *
  22 * This file implements early detection/parsing of Remapping Devices
  23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24 * tables.
  25 *
  26 * These routines are used by both DMA-remapping and Interrupt-remapping
  27 */
  28
  29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
  30
  31#include <linux/pci.h>
  32#include <linux/dmar.h>
  33#include <linux/iova.h>
  34#include <linux/intel-iommu.h>
  35#include <linux/timer.h>
  36#include <linux/irq.h>
  37#include <linux/interrupt.h>
  38#include <linux/tboot.h>
  39#include <linux/dmi.h>
  40#include <linux/slab.h>
  41#include <asm/irq_remapping.h>
  42#include <asm/iommu_table.h>
  43
  44#include "irq_remapping.h"
  45
  46/* No locks are needed as DMA remapping hardware unit
  47 * list is constructed at boot time and hotplug of
  48 * these units are not supported by the architecture.
  49 */
  50LIST_HEAD(dmar_drhd_units);
  51
  52struct acpi_table_header * __initdata dmar_tbl;
  53static acpi_size dmar_tbl_size;
  54
  55static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  56{
  57        /*
  58         * add INCLUDE_ALL at the tail, so scan the list will find it at
  59         * the very end.
  60         */
  61        if (drhd->include_all)
  62                list_add_tail(&drhd->list, &dmar_drhd_units);
  63        else
  64                list_add(&drhd->list, &dmar_drhd_units);
  65}
  66
  67static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  68                                           struct pci_dev **dev, u16 segment)
  69{
  70        struct pci_bus *bus;
  71        struct pci_dev *pdev = NULL;
  72        struct acpi_dmar_pci_path *path;
  73        int count;
  74
  75        bus = pci_find_bus(segment, scope->bus);
  76        path = (struct acpi_dmar_pci_path *)(scope + 1);
  77        count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  78                / sizeof(struct acpi_dmar_pci_path);
  79
  80        while (count) {
  81                if (pdev)
  82                        pci_dev_put(pdev);
  83                /*
  84                 * Some BIOSes list non-exist devices in DMAR table, just
  85                 * ignore it
  86                 */
  87                if (!bus) {
  88                        pr_warn("Device scope bus [%d] not found\n", scope->bus);
  89                        break;
  90                }
  91                pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  92                if (!pdev) {
  93                        /* warning will be printed below */
  94                        break;
  95                }
  96                path ++;
  97                count --;
  98                bus = pdev->subordinate;
  99        }
 100        if (!pdev) {
 101                pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
 102                        segment, scope->bus, path->dev, path->fn);
 103                *dev = NULL;
 104                return 0;
 105        }
 106        if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 107                        pdev->subordinate) || (scope->entry_type == \
 108                        ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 109                pci_dev_put(pdev);
 110                pr_warn("Device scope type does not match for %s\n",
 111                        pci_name(pdev));
 112                return -EINVAL;
 113        }
 114        *dev = pdev;
 115        return 0;
 116}
 117
 118int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 119                                struct pci_dev ***devices, u16 segment)
 120{
 121        struct acpi_dmar_device_scope *scope;
 122        void * tmp = start;
 123        int index;
 124        int ret;
 125
 126        *cnt = 0;
 127        while (start < end) {
 128                scope = start;
 129                if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 130                    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 131                        (*cnt)++;
 132                else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 133                        scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 134                        pr_warn("Unsupported device scope\n");
 135                }
 136                start += scope->length;
 137        }
 138        if (*cnt == 0)
 139                return 0;
 140
 141        *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 142        if (!*devices)
 143                return -ENOMEM;
 144
 145        start = tmp;
 146        index = 0;
 147        while (start < end) {
 148                scope = start;
 149                if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 150                    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 151                        ret = dmar_parse_one_dev_scope(scope,
 152                                &(*devices)[index], segment);
 153                        if (ret) {
 154                                kfree(*devices);
 155                                return ret;
 156                        }
 157                        index ++;
 158                }
 159                start += scope->length;
 160        }
 161
 162        return 0;
 163}
 164
 165/**
 166 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 167 * structure which uniquely represent one DMA remapping hardware unit
 168 * present in the platform
 169 */
 170static int __init
 171dmar_parse_one_drhd(struct acpi_dmar_header *header)
 172{
 173        struct acpi_dmar_hardware_unit *drhd;
 174        struct dmar_drhd_unit *dmaru;
 175        int ret = 0;
 176
 177        drhd = (struct acpi_dmar_hardware_unit *)header;
 178        dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 179        if (!dmaru)
 180                return -ENOMEM;
 181
 182        dmaru->hdr = header;
 183        dmaru->reg_base_addr = drhd->address;
 184        dmaru->segment = drhd->segment;
 185        dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 186
 187        ret = alloc_iommu(dmaru);
 188        if (ret) {
 189                kfree(dmaru);
 190                return ret;
 191        }
 192        dmar_register_drhd_unit(dmaru);
 193        return 0;
 194}
 195
 196static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 197{
 198        struct acpi_dmar_hardware_unit *drhd;
 199        int ret = 0;
 200
 201        drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 202
 203        if (dmaru->include_all)
 204                return 0;
 205
 206        ret = dmar_parse_dev_scope((void *)(drhd + 1),
 207                                ((void *)drhd) + drhd->header.length,
 208                                &dmaru->devices_cnt, &dmaru->devices,
 209                                drhd->segment);
 210        if (ret) {
 211                list_del(&dmaru->list);
 212                kfree(dmaru);
 213        }
 214        return ret;
 215}
 216
 217#ifdef CONFIG_ACPI_NUMA
 218static int __init
 219dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 220{
 221        struct acpi_dmar_rhsa *rhsa;
 222        struct dmar_drhd_unit *drhd;
 223
 224        rhsa = (struct acpi_dmar_rhsa *)header;
 225        for_each_drhd_unit(drhd) {
 226                if (drhd->reg_base_addr == rhsa->base_address) {
 227                        int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 228
 229                        if (!node_online(node))
 230                                node = -1;
 231                        drhd->iommu->node = node;
 232                        return 0;
 233                }
 234        }
 235        WARN_TAINT(
 236                1, TAINT_FIRMWARE_WORKAROUND,
 237                "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 238                "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 239                drhd->reg_base_addr,
 240                dmi_get_system_info(DMI_BIOS_VENDOR),
 241                dmi_get_system_info(DMI_BIOS_VERSION),
 242                dmi_get_system_info(DMI_PRODUCT_VERSION));
 243
 244        return 0;
 245}
 246#endif
 247
 248static void __init
 249dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 250{
 251        struct acpi_dmar_hardware_unit *drhd;
 252        struct acpi_dmar_reserved_memory *rmrr;
 253        struct acpi_dmar_atsr *atsr;
 254        struct acpi_dmar_rhsa *rhsa;
 255
 256        switch (header->type) {
 257        case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 258                drhd = container_of(header, struct acpi_dmar_hardware_unit,
 259                                    header);
 260                pr_info("DRHD base: %#016Lx flags: %#x\n",
 261                        (unsigned long long)drhd->address, drhd->flags);
 262                break;
 263        case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 264                rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 265                                    header);
 266                pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 267                        (unsigned long long)rmrr->base_address,
 268                        (unsigned long long)rmrr->end_address);
 269                break;
 270        case ACPI_DMAR_TYPE_ATSR:
 271                atsr = container_of(header, struct acpi_dmar_atsr, header);
 272                pr_info("ATSR flags: %#x\n", atsr->flags);
 273                break;
 274        case ACPI_DMAR_HARDWARE_AFFINITY:
 275                rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 276                pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 277                       (unsigned long long)rhsa->base_address,
 278                       rhsa->proximity_domain);
 279                break;
 280        }
 281}
 282
 283/**
 284 * dmar_table_detect - checks to see if the platform supports DMAR devices
 285 */
 286static int __init dmar_table_detect(void)
 287{
 288        acpi_status status = AE_OK;
 289
 290        /* if we could find DMAR table, then there are DMAR devices */
 291        status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 292                                (struct acpi_table_header **)&dmar_tbl,
 293                                &dmar_tbl_size);
 294
 295        if (ACPI_SUCCESS(status) && !dmar_tbl) {
 296                pr_warn("Unable to map DMAR\n");
 297                status = AE_NOT_FOUND;
 298        }
 299
 300        return (ACPI_SUCCESS(status) ? 1 : 0);
 301}
 302
 303/**
 304 * parse_dmar_table - parses the DMA reporting table
 305 */
 306static int __init
 307parse_dmar_table(void)
 308{
 309        struct acpi_table_dmar *dmar;
 310        struct acpi_dmar_header *entry_header;
 311        int ret = 0;
 312
 313        /*
 314         * Do it again, earlier dmar_tbl mapping could be mapped with
 315         * fixed map.
 316         */
 317        dmar_table_detect();
 318
 319        /*
 320         * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 321         * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 322         */
 323        dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 324
 325        dmar = (struct acpi_table_dmar *)dmar_tbl;
 326        if (!dmar)
 327                return -ENODEV;
 328
 329        if (dmar->width < PAGE_SHIFT - 1) {
 330                pr_warn("Invalid DMAR haw\n");
 331                return -EINVAL;
 332        }
 333
 334        pr_info("Host address width %d\n", dmar->width + 1);
 335
 336        entry_header = (struct acpi_dmar_header *)(dmar + 1);
 337        while (((unsigned long)entry_header) <
 338                        (((unsigned long)dmar) + dmar_tbl->length)) {
 339                /* Avoid looping forever on bad ACPI tables */
 340                if (entry_header->length == 0) {
 341                        pr_warn("Invalid 0-length structure\n");
 342                        ret = -EINVAL;
 343                        break;
 344                }
 345
 346                dmar_table_print_dmar_entry(entry_header);
 347
 348                switch (entry_header->type) {
 349                case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 350                        ret = dmar_parse_one_drhd(entry_header);
 351                        break;
 352                case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 353                        ret = dmar_parse_one_rmrr(entry_header);
 354                        break;
 355                case ACPI_DMAR_TYPE_ATSR:
 356                        ret = dmar_parse_one_atsr(entry_header);
 357                        break;
 358                case ACPI_DMAR_HARDWARE_AFFINITY:
 359#ifdef CONFIG_ACPI_NUMA
 360                        ret = dmar_parse_one_rhsa(entry_header);
 361#endif
 362                        break;
 363                default:
 364                        pr_warn("Unknown DMAR structure type %d\n",
 365                                entry_header->type);
 366                        ret = 0; /* for forward compatibility */
 367                        break;
 368                }
 369                if (ret)
 370                        break;
 371
 372                entry_header = ((void *)entry_header + entry_header->length);
 373        }
 374        return ret;
 375}
 376
 377static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 378                          struct pci_dev *dev)
 379{
 380        int index;
 381
 382        while (dev) {
 383                for (index = 0; index < cnt; index++)
 384                        if (dev == devices[index])
 385                                return 1;
 386
 387                /* Check our parent */
 388                dev = dev->bus->self;
 389        }
 390
 391        return 0;
 392}
 393
 394struct dmar_drhd_unit *
 395dmar_find_matched_drhd_unit(struct pci_dev *dev)
 396{
 397        struct dmar_drhd_unit *dmaru = NULL;
 398        struct acpi_dmar_hardware_unit *drhd;
 399
 400        dev = pci_physfn(dev);
 401
 402        list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 403                drhd = container_of(dmaru->hdr,
 404                                    struct acpi_dmar_hardware_unit,
 405                                    header);
 406
 407                if (dmaru->include_all &&
 408                    drhd->segment == pci_domain_nr(dev->bus))
 409                        return dmaru;
 410
 411                if (dmar_pci_device_match(dmaru->devices,
 412                                          dmaru->devices_cnt, dev))
 413                        return dmaru;
 414        }
 415
 416        return NULL;
 417}
 418
 419int __init dmar_dev_scope_init(void)
 420{
 421        static int dmar_dev_scope_initialized;
 422        struct dmar_drhd_unit *drhd, *drhd_n;
 423        int ret = -ENODEV;
 424
 425        if (dmar_dev_scope_initialized)
 426                return dmar_dev_scope_initialized;
 427
 428        if (list_empty(&dmar_drhd_units))
 429                goto fail;
 430
 431        list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 432                ret = dmar_parse_dev(drhd);
 433                if (ret)
 434                        goto fail;
 435        }
 436
 437        ret = dmar_parse_rmrr_atsr_dev();
 438        if (ret)
 439                goto fail;
 440
 441        dmar_dev_scope_initialized = 1;
 442        return 0;
 443
 444fail:
 445        dmar_dev_scope_initialized = ret;
 446        return ret;
 447}
 448
 449
 450int __init dmar_table_init(void)
 451{
 452        static int dmar_table_initialized;
 453        int ret;
 454
 455        if (dmar_table_initialized)
 456                return 0;
 457
 458        dmar_table_initialized = 1;
 459
 460        ret = parse_dmar_table();
 461        if (ret) {
 462                if (ret != -ENODEV)
 463                        pr_info("parse DMAR table failure.\n");
 464                return ret;
 465        }
 466
 467        if (list_empty(&dmar_drhd_units)) {
 468                pr_info("No DMAR devices found\n");
 469                return -ENODEV;
 470        }
 471
 472        return 0;
 473}
 474
 475static void warn_invalid_dmar(u64 addr, const char *message)
 476{
 477        WARN_TAINT_ONCE(
 478                1, TAINT_FIRMWARE_WORKAROUND,
 479                "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 480                "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 481                addr, message,
 482                dmi_get_system_info(DMI_BIOS_VENDOR),
 483                dmi_get_system_info(DMI_BIOS_VERSION),
 484                dmi_get_system_info(DMI_PRODUCT_VERSION));
 485}
 486
 487int __init check_zero_address(void)
 488{
 489        struct acpi_table_dmar *dmar;
 490        struct acpi_dmar_header *entry_header;
 491        struct acpi_dmar_hardware_unit *drhd;
 492
 493        dmar = (struct acpi_table_dmar *)dmar_tbl;
 494        entry_header = (struct acpi_dmar_header *)(dmar + 1);
 495
 496        while (((unsigned long)entry_header) <
 497                        (((unsigned long)dmar) + dmar_tbl->length)) {
 498                /* Avoid looping forever on bad ACPI tables */
 499                if (entry_header->length == 0) {
 500                        pr_warn("Invalid 0-length structure\n");
 501                        return 0;
 502                }
 503
 504                if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 505                        void __iomem *addr;
 506                        u64 cap, ecap;
 507
 508                        drhd = (void *)entry_header;
 509                        if (!drhd->address) {
 510                                warn_invalid_dmar(0, "");
 511                                goto failed;
 512                        }
 513
 514                        addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 515                        if (!addr ) {
 516                                printk("IOMMU: can't validate: %llx\n", drhd->address);
 517                                goto failed;
 518                        }
 519                        cap = dmar_readq(addr + DMAR_CAP_REG);
 520                        ecap = dmar_readq(addr + DMAR_ECAP_REG);
 521                        early_iounmap(addr, VTD_PAGE_SIZE);
 522                        if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 523                                warn_invalid_dmar(drhd->address,
 524                                                  " returns all ones");
 525                                goto failed;
 526                        }
 527                }
 528
 529                entry_header = ((void *)entry_header + entry_header->length);
 530        }
 531        return 1;
 532
 533failed:
 534        return 0;
 535}
 536
 537int __init detect_intel_iommu(void)
 538{
 539        int ret;
 540
 541        ret = dmar_table_detect();
 542        if (ret)
 543                ret = check_zero_address();
 544        {
 545                struct acpi_table_dmar *dmar;
 546
 547                dmar = (struct acpi_table_dmar *) dmar_tbl;
 548
 549                if (ret && irq_remapping_enabled && cpu_has_x2apic &&
 550                    dmar->flags & 0x1)
 551                        pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
 552
 553                if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 554                        iommu_detected = 1;
 555                        /* Make sure ACS will be enabled */
 556                        pci_request_acs();
 557                }
 558
 559#ifdef CONFIG_X86
 560                if (ret)
 561                        x86_init.iommu.iommu_init = intel_iommu_init;
 562#endif
 563        }
 564        early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 565        dmar_tbl = NULL;
 566
 567        return ret ? 1 : -ENODEV;
 568}
 569
 570
 571static void unmap_iommu(struct intel_iommu *iommu)
 572{
 573        iounmap(iommu->reg);
 574        release_mem_region(iommu->reg_phys, iommu->reg_size);
 575}
 576
 577/**
 578 * map_iommu: map the iommu's registers
 579 * @iommu: the iommu to map
 580 * @phys_addr: the physical address of the base resgister
 581 *
 582 * Memory map the iommu's registers.  Start w/ a single page, and
 583 * possibly expand if that turns out to be insufficent.
 584 */
 585static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 586{
 587        int map_size, err=0;
 588
 589        iommu->reg_phys = phys_addr;
 590        iommu->reg_size = VTD_PAGE_SIZE;
 591
 592        if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 593                pr_err("IOMMU: can't reserve memory\n");
 594                err = -EBUSY;
 595                goto out;
 596        }
 597
 598        iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 599        if (!iommu->reg) {
 600                pr_err("IOMMU: can't map the region\n");
 601                err = -ENOMEM;
 602                goto release;
 603        }
 604
 605        iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 606        iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 607
 608        if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 609                err = -EINVAL;
 610                warn_invalid_dmar(phys_addr, " returns all ones");
 611                goto unmap;
 612        }
 613
 614        /* the registers might be more than one page */
 615        map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 616                         cap_max_fault_reg_offset(iommu->cap));
 617        map_size = VTD_PAGE_ALIGN(map_size);
 618        if (map_size > iommu->reg_size) {
 619                iounmap(iommu->reg);
 620                release_mem_region(iommu->reg_phys, iommu->reg_size);
 621                iommu->reg_size = map_size;
 622                if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 623                                        iommu->name)) {
 624                        pr_err("IOMMU: can't reserve memory\n");
 625                        err = -EBUSY;
 626                        goto out;
 627                }
 628                iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 629                if (!iommu->reg) {
 630                        pr_err("IOMMU: can't map the region\n");
 631                        err = -ENOMEM;
 632                        goto release;
 633                }
 634        }
 635        err = 0;
 636        goto out;
 637
 638unmap:
 639        iounmap(iommu->reg);
 640release:
 641        release_mem_region(iommu->reg_phys, iommu->reg_size);
 642out:
 643        return err;
 644}
 645
 646int alloc_iommu(struct dmar_drhd_unit *drhd)
 647{
 648        struct intel_iommu *iommu;
 649        u32 ver, sts;
 650        static int iommu_allocated = 0;
 651        int agaw = 0;
 652        int msagaw = 0;
 653        int err;
 654
 655        if (!drhd->reg_base_addr) {
 656                warn_invalid_dmar(0, "");
 657                return -EINVAL;
 658        }
 659
 660        iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 661        if (!iommu)
 662                return -ENOMEM;
 663
 664        iommu->seq_id = iommu_allocated++;
 665        sprintf (iommu->name, "dmar%d", iommu->seq_id);
 666
 667        err = map_iommu(iommu, drhd->reg_base_addr);
 668        if (err) {
 669                pr_err("IOMMU: failed to map %s\n", iommu->name);
 670                goto error;
 671        }
 672
 673        err = -EINVAL;
 674        agaw = iommu_calculate_agaw(iommu);
 675        if (agaw < 0) {
 676                pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
 677                        iommu->seq_id);
 678                goto err_unmap;
 679        }
 680        msagaw = iommu_calculate_max_sagaw(iommu);
 681        if (msagaw < 0) {
 682                pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 683                        iommu->seq_id);
 684                goto err_unmap;
 685        }
 686        iommu->agaw = agaw;
 687        iommu->msagaw = msagaw;
 688
 689        iommu->node = -1;
 690
 691        ver = readl(iommu->reg + DMAR_VER_REG);
 692        pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 693                iommu->seq_id,
 694                (unsigned long long)drhd->reg_base_addr,
 695                DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 696                (unsigned long long)iommu->cap,
 697                (unsigned long long)iommu->ecap);
 698
 699        /* Reflect status in gcmd */
 700        sts = readl(iommu->reg + DMAR_GSTS_REG);
 701        if (sts & DMA_GSTS_IRES)
 702                iommu->gcmd |= DMA_GCMD_IRE;
 703        if (sts & DMA_GSTS_TES)
 704                iommu->gcmd |= DMA_GCMD_TE;
 705        if (sts & DMA_GSTS_QIES)
 706                iommu->gcmd |= DMA_GCMD_QIE;
 707
 708        raw_spin_lock_init(&iommu->register_lock);
 709
 710        drhd->iommu = iommu;
 711        return 0;
 712
 713 err_unmap:
 714        unmap_iommu(iommu);
 715 error:
 716        kfree(iommu);
 717        return err;
 718}
 719
 720void free_iommu(struct intel_iommu *iommu)
 721{
 722        if (!iommu)
 723                return;
 724
 725        free_dmar_iommu(iommu);
 726
 727        if (iommu->reg)
 728                unmap_iommu(iommu);
 729
 730        kfree(iommu);
 731}
 732
 733/*
 734 * Reclaim all the submitted descriptors which have completed its work.
 735 */
 736static inline void reclaim_free_desc(struct q_inval *qi)
 737{
 738        while (qi->desc_status[qi->free_tail] == QI_DONE ||
 739               qi->desc_status[qi->free_tail] == QI_ABORT) {
 740                qi->desc_status[qi->free_tail] = QI_FREE;
 741                qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 742                qi->free_cnt++;
 743        }
 744}
 745
 746static int qi_check_fault(struct intel_iommu *iommu, int index)
 747{
 748        u32 fault;
 749        int head, tail;
 750        struct q_inval *qi = iommu->qi;
 751        int wait_index = (index + 1) % QI_LENGTH;
 752
 753        if (qi->desc_status[wait_index] == QI_ABORT)
 754                return -EAGAIN;
 755
 756        fault = readl(iommu->reg + DMAR_FSTS_REG);
 757
 758        /*
 759         * If IQE happens, the head points to the descriptor associated
 760         * with the error. No new descriptors are fetched until the IQE
 761         * is cleared.
 762         */
 763        if (fault & DMA_FSTS_IQE) {
 764                head = readl(iommu->reg + DMAR_IQH_REG);
 765                if ((head >> DMAR_IQ_SHIFT) == index) {
 766                        pr_err("VT-d detected invalid descriptor: "
 767                                "low=%llx, high=%llx\n",
 768                                (unsigned long long)qi->desc[index].low,
 769                                (unsigned long long)qi->desc[index].high);
 770                        memcpy(&qi->desc[index], &qi->desc[wait_index],
 771                                        sizeof(struct qi_desc));
 772                        __iommu_flush_cache(iommu, &qi->desc[index],
 773                                        sizeof(struct qi_desc));
 774                        writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 775                        return -EINVAL;
 776                }
 777        }
 778
 779        /*
 780         * If ITE happens, all pending wait_desc commands are aborted.
 781         * No new descriptors are fetched until the ITE is cleared.
 782         */
 783        if (fault & DMA_FSTS_ITE) {
 784                head = readl(iommu->reg + DMAR_IQH_REG);
 785                head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 786                head |= 1;
 787                tail = readl(iommu->reg + DMAR_IQT_REG);
 788                tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 789
 790                writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 791
 792                do {
 793                        if (qi->desc_status[head] == QI_IN_USE)
 794                                qi->desc_status[head] = QI_ABORT;
 795                        head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 796                } while (head != tail);
 797
 798                if (qi->desc_status[wait_index] == QI_ABORT)
 799                        return -EAGAIN;
 800        }
 801
 802        if (fault & DMA_FSTS_ICE)
 803                writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 804
 805        return 0;
 806}
 807
 808/*
 809 * Submit the queued invalidation descriptor to the remapping
 810 * hardware unit and wait for its completion.
 811 */
 812int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 813{
 814        int rc;
 815        struct q_inval *qi = iommu->qi;
 816        struct qi_desc *hw, wait_desc;
 817        int wait_index, index;
 818        unsigned long flags;
 819
 820        if (!qi)
 821                return 0;
 822
 823        hw = qi->desc;
 824
 825restart:
 826        rc = 0;
 827
 828        raw_spin_lock_irqsave(&qi->q_lock, flags);
 829        while (qi->free_cnt < 3) {
 830                raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 831                cpu_relax();
 832                raw_spin_lock_irqsave(&qi->q_lock, flags);
 833        }
 834
 835        index = qi->free_head;
 836        wait_index = (index + 1) % QI_LENGTH;
 837
 838        qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 839
 840        hw[index] = *desc;
 841
 842        wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 843                        QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 844        wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 845
 846        hw[wait_index] = wait_desc;
 847
 848        __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 849        __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 850
 851        qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 852        qi->free_cnt -= 2;
 853
 854        /*
 855         * update the HW tail register indicating the presence of
 856         * new descriptors.
 857         */
 858        writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 859
 860        while (qi->desc_status[wait_index] != QI_DONE) {
 861                /*
 862                 * We will leave the interrupts disabled, to prevent interrupt
 863                 * context to queue another cmd while a cmd is already submitted
 864                 * and waiting for completion on this cpu. This is to avoid
 865                 * a deadlock where the interrupt context can wait indefinitely
 866                 * for free slots in the queue.
 867                 */
 868                rc = qi_check_fault(iommu, index);
 869                if (rc)
 870                        break;
 871
 872                raw_spin_unlock(&qi->q_lock);
 873                cpu_relax();
 874                raw_spin_lock(&qi->q_lock);
 875        }
 876
 877        qi->desc_status[index] = QI_DONE;
 878
 879        reclaim_free_desc(qi);
 880        raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 881
 882        if (rc == -EAGAIN)
 883                goto restart;
 884
 885        return rc;
 886}
 887
 888/*
 889 * Flush the global interrupt entry cache.
 890 */
 891void qi_global_iec(struct intel_iommu *iommu)
 892{
 893        struct qi_desc desc;
 894
 895        desc.low = QI_IEC_TYPE;
 896        desc.high = 0;
 897
 898        /* should never fail */
 899        qi_submit_sync(&desc, iommu);
 900}
 901
 902void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 903                      u64 type)
 904{
 905        struct qi_desc desc;
 906
 907        desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 908                        | QI_CC_GRAN(type) | QI_CC_TYPE;
 909        desc.high = 0;
 910
 911        qi_submit_sync(&desc, iommu);
 912}
 913
 914void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 915                    unsigned int size_order, u64 type)
 916{
 917        u8 dw = 0, dr = 0;
 918
 919        struct qi_desc desc;
 920        int ih = 0;
 921
 922        if (cap_write_drain(iommu->cap))
 923                dw = 1;
 924
 925        if (cap_read_drain(iommu->cap))
 926                dr = 1;
 927
 928        desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 929                | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 930        desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 931                | QI_IOTLB_AM(size_order);
 932
 933        qi_submit_sync(&desc, iommu);
 934}
 935
 936void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 937                        u64 addr, unsigned mask)
 938{
 939        struct qi_desc desc;
 940
 941        if (mask) {
 942                BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 943                addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 944                desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 945        } else
 946                desc.high = QI_DEV_IOTLB_ADDR(addr);
 947
 948        if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 949                qdep = 0;
 950
 951        desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 952                   QI_DIOTLB_TYPE;
 953
 954        qi_submit_sync(&desc, iommu);
 955}
 956
 957/*
 958 * Disable Queued Invalidation interface.
 959 */
 960void dmar_disable_qi(struct intel_iommu *iommu)
 961{
 962        unsigned long flags;
 963        u32 sts;
 964        cycles_t start_time = get_cycles();
 965
 966        if (!ecap_qis(iommu->ecap))
 967                return;
 968
 969        raw_spin_lock_irqsave(&iommu->register_lock, flags);
 970
 971        sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 972        if (!(sts & DMA_GSTS_QIES))
 973                goto end;
 974
 975        /*
 976         * Give a chance to HW to complete the pending invalidation requests.
 977         */
 978        while ((readl(iommu->reg + DMAR_IQT_REG) !=
 979                readl(iommu->reg + DMAR_IQH_REG)) &&
 980                (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 981                cpu_relax();
 982
 983        iommu->gcmd &= ~DMA_GCMD_QIE;
 984        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 985
 986        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 987                      !(sts & DMA_GSTS_QIES), sts);
 988end:
 989        raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
 990}
 991
 992/*
 993 * Enable queued invalidation.
 994 */
 995static void __dmar_enable_qi(struct intel_iommu *iommu)
 996{
 997        u32 sts;
 998        unsigned long flags;
 999        struct q_inval *qi = iommu->qi;
1000
1001        qi->free_head = qi->free_tail = 0;
1002        qi->free_cnt = QI_LENGTH;
1003
1004        raw_spin_lock_irqsave(&iommu->register_lock, flags);
1005
1006        /* write zero to the tail reg */
1007        writel(0, iommu->reg + DMAR_IQT_REG);
1008
1009        dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1010
1011        iommu->gcmd |= DMA_GCMD_QIE;
1012        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1013
1014        /* Make sure hardware complete it */
1015        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1016
1017        raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1018}
1019
1020/*
1021 * Enable Queued Invalidation interface. This is a must to support
1022 * interrupt-remapping. Also used by DMA-remapping, which replaces
1023 * register based IOTLB invalidation.
1024 */
1025int dmar_enable_qi(struct intel_iommu *iommu)
1026{
1027        struct q_inval *qi;
1028        struct page *desc_page;
1029
1030        if (!ecap_qis(iommu->ecap))
1031                return -ENOENT;
1032
1033        /*
1034         * queued invalidation is already setup and enabled.
1035         */
1036        if (iommu->qi)
1037                return 0;
1038
1039        iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1040        if (!iommu->qi)
1041                return -ENOMEM;
1042
1043        qi = iommu->qi;
1044
1045
1046        desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1047        if (!desc_page) {
1048                kfree(qi);
1049                iommu->qi = 0;
1050                return -ENOMEM;
1051        }
1052
1053        qi->desc = page_address(desc_page);
1054
1055        qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1056        if (!qi->desc_status) {
1057                free_page((unsigned long) qi->desc);
1058                kfree(qi);
1059                iommu->qi = 0;
1060                return -ENOMEM;
1061        }
1062
1063        qi->free_head = qi->free_tail = 0;
1064        qi->free_cnt = QI_LENGTH;
1065
1066        raw_spin_lock_init(&qi->q_lock);
1067
1068        __dmar_enable_qi(iommu);
1069
1070        return 0;
1071}
1072
1073/* iommu interrupt handling. Most stuff are MSI-like. */
1074
1075enum faulttype {
1076        DMA_REMAP,
1077        INTR_REMAP,
1078        UNKNOWN,
1079};
1080
1081static const char *dma_remap_fault_reasons[] =
1082{
1083        "Software",
1084        "Present bit in root entry is clear",
1085        "Present bit in context entry is clear",
1086        "Invalid context entry",
1087        "Access beyond MGAW",
1088        "PTE Write access is not set",
1089        "PTE Read access is not set",
1090        "Next page table ptr is invalid",
1091        "Root table address invalid",
1092        "Context table ptr is invalid",
1093        "non-zero reserved fields in RTP",
1094        "non-zero reserved fields in CTP",
1095        "non-zero reserved fields in PTE",
1096        "PCE for translation request specifies blocking",
1097};
1098
1099static const char *irq_remap_fault_reasons[] =
1100{
1101        "Detected reserved fields in the decoded interrupt-remapped request",
1102        "Interrupt index exceeded the interrupt-remapping table size",
1103        "Present field in the IRTE entry is clear",
1104        "Error accessing interrupt-remapping table pointed by IRTA_REG",
1105        "Detected reserved fields in the IRTE entry",
1106        "Blocked a compatibility format interrupt request",
1107        "Blocked an interrupt request due to source-id verification failure",
1108};
1109
1110#define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1111
1112const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1113{
1114        if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1115                                        ARRAY_SIZE(irq_remap_fault_reasons))) {
1116                *fault_type = INTR_REMAP;
1117                return irq_remap_fault_reasons[fault_reason - 0x20];
1118        } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1119                *fault_type = DMA_REMAP;
1120                return dma_remap_fault_reasons[fault_reason];
1121        } else {
1122                *fault_type = UNKNOWN;
1123                return "Unknown";
1124        }
1125}
1126
1127void dmar_msi_unmask(struct irq_data *data)
1128{
1129        struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1130        unsigned long flag;
1131
1132        /* unmask it */
1133        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1134        writel(0, iommu->reg + DMAR_FECTL_REG);
1135        /* Read a reg to force flush the post write */
1136        readl(iommu->reg + DMAR_FECTL_REG);
1137        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1138}
1139
1140void dmar_msi_mask(struct irq_data *data)
1141{
1142        unsigned long flag;
1143        struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1144
1145        /* mask it */
1146        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1147        writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1148        /* Read a reg to force flush the post write */
1149        readl(iommu->reg + DMAR_FECTL_REG);
1150        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1151}
1152
1153void dmar_msi_write(int irq, struct msi_msg *msg)
1154{
1155        struct intel_iommu *iommu = irq_get_handler_data(irq);
1156        unsigned long flag;
1157
1158        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1159        writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1160        writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1161        writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1162        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1163}
1164
1165void dmar_msi_read(int irq, struct msi_msg *msg)
1166{
1167        struct intel_iommu *iommu = irq_get_handler_data(irq);
1168        unsigned long flag;
1169
1170        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1171        msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1172        msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1173        msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1174        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1175}
1176
1177static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1178                u8 fault_reason, u16 source_id, unsigned long long addr)
1179{
1180        const char *reason;
1181        int fault_type;
1182
1183        reason = dmar_get_fault_reason(fault_reason, &fault_type);
1184
1185        if (fault_type == INTR_REMAP)
1186                pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1187                       "fault index %llx\n"
1188                        "INTR-REMAP:[fault reason %02d] %s\n",
1189                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1190                        PCI_FUNC(source_id & 0xFF), addr >> 48,
1191                        fault_reason, reason);
1192        else
1193                pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1194                       "fault addr %llx \n"
1195                       "DMAR:[fault reason %02d] %s\n",
1196                       (type ? "DMA Read" : "DMA Write"),
1197                       (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1198                       PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1199        return 0;
1200}
1201
1202#define PRIMARY_FAULT_REG_LEN (16)
1203irqreturn_t dmar_fault(int irq, void *dev_id)
1204{
1205        struct intel_iommu *iommu = dev_id;
1206        int reg, fault_index;
1207        u32 fault_status;
1208        unsigned long flag;
1209
1210        raw_spin_lock_irqsave(&iommu->register_lock, flag);
1211        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1212        if (fault_status)
1213                pr_err("DRHD: handling fault status reg %x\n", fault_status);
1214
1215        /* TBD: ignore advanced fault log currently */
1216        if (!(fault_status & DMA_FSTS_PPF))
1217                goto unlock_exit;
1218
1219        fault_index = dma_fsts_fault_record_index(fault_status);
1220        reg = cap_fault_reg_offset(iommu->cap);
1221        while (1) {
1222                u8 fault_reason;
1223                u16 source_id;
1224                u64 guest_addr;
1225                int type;
1226                u32 data;
1227
1228                /* highest 32 bits */
1229                data = readl(iommu->reg + reg +
1230                                fault_index * PRIMARY_FAULT_REG_LEN + 12);
1231                if (!(data & DMA_FRCD_F))
1232                        break;
1233
1234                fault_reason = dma_frcd_fault_reason(data);
1235                type = dma_frcd_type(data);
1236
1237                data = readl(iommu->reg + reg +
1238                                fault_index * PRIMARY_FAULT_REG_LEN + 8);
1239                source_id = dma_frcd_source_id(data);
1240
1241                guest_addr = dmar_readq(iommu->reg + reg +
1242                                fault_index * PRIMARY_FAULT_REG_LEN);
1243                guest_addr = dma_frcd_page_addr(guest_addr);
1244                /* clear the fault */
1245                writel(DMA_FRCD_F, iommu->reg + reg +
1246                        fault_index * PRIMARY_FAULT_REG_LEN + 12);
1247
1248                raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1249
1250                dmar_fault_do_one(iommu, type, fault_reason,
1251                                source_id, guest_addr);
1252
1253                fault_index++;
1254                if (fault_index >= cap_num_fault_regs(iommu->cap))
1255                        fault_index = 0;
1256                raw_spin_lock_irqsave(&iommu->register_lock, flag);
1257        }
1258
1259        writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1260
1261unlock_exit:
1262        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1263        return IRQ_HANDLED;
1264}
1265
1266int dmar_set_interrupt(struct intel_iommu *iommu)
1267{
1268        int irq, ret;
1269
1270        /*
1271         * Check if the fault interrupt is already initialized.
1272         */
1273        if (iommu->irq)
1274                return 0;
1275
1276        irq = create_irq();
1277        if (!irq) {
1278                pr_err("IOMMU: no free vectors\n");
1279                return -EINVAL;
1280        }
1281
1282        irq_set_handler_data(irq, iommu);
1283        iommu->irq = irq;
1284
1285        ret = arch_setup_dmar_msi(irq);
1286        if (ret) {
1287                irq_set_handler_data(irq, NULL);
1288                iommu->irq = 0;
1289                destroy_irq(irq);
1290                return ret;
1291        }
1292
1293        ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1294        if (ret)
1295                pr_err("IOMMU: can't request irq\n");
1296        return ret;
1297}
1298
1299int __init enable_drhd_fault_handling(void)
1300{
1301        struct dmar_drhd_unit *drhd;
1302
1303        /*
1304         * Enable fault control interrupt.
1305         */
1306        for_each_drhd_unit(drhd) {
1307                int ret;
1308                struct intel_iommu *iommu = drhd->iommu;
1309                u32 fault_status;
1310                ret = dmar_set_interrupt(iommu);
1311
1312                if (ret) {
1313                        pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1314                               (unsigned long long)drhd->reg_base_addr, ret);
1315                        return -1;
1316                }
1317
1318                /*
1319                 * Clear any previous faults.
1320                 */
1321                dmar_fault(iommu->irq, iommu);
1322                fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1323                writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1324        }
1325
1326        return 0;
1327}
1328
1329/*
1330 * Re-enable Queued Invalidation interface.
1331 */
1332int dmar_reenable_qi(struct intel_iommu *iommu)
1333{
1334        if (!ecap_qis(iommu->ecap))
1335                return -ENOENT;
1336
1337        if (!iommu->qi)
1338                return -ENOENT;
1339
1340        /*
1341         * First disable queued invalidation.
1342         */
1343        dmar_disable_qi(iommu);
1344        /*
1345         * Then enable queued invalidation again. Since there is no pending
1346         * invalidation requests now, it's safe to re-enable queued
1347         * invalidation.
1348         */
1349        __dmar_enable_qi(iommu);
1350
1351        return 0;
1352}
1353
1354/*
1355 * Check interrupt remapping support in DMAR table description.
1356 */
1357int __init dmar_ir_support(void)
1358{
1359        struct acpi_table_dmar *dmar;
1360        dmar = (struct acpi_table_dmar *)dmar_tbl;
1361        if (!dmar)
1362                return 0;
1363        return dmar->flags & 0x1;
1364}
1365IOMMU_INIT_POST(detect_intel_iommu);
1366
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.