linux/drivers/pci/dmar.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2006, Intel Corporation.
   3 *
   4 * This program is free software; you can redistribute it and/or modify it
   5 * under the terms and conditions of the GNU General Public License,
   6 * version 2, as published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope it will be useful, but WITHOUT
   9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 * more details.
  12 *
  13 * You should have received a copy of the GNU General Public License along with
  14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15 * Place - Suite 330, Boston, MA 02111-1307 USA.
  16 *
  17 * Copyright (C) 2006-2008 Intel Corporation
  18 * Author: Ashok Raj <ashok.raj@intel.com>
  19 * Author: Shaohua Li <shaohua.li@intel.com>
  20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21 *
  22 * This file implements early detection/parsing of Remapping Devices
  23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24 * tables.
  25 *
  26 * These routines are used by both DMA-remapping and Interrupt-remapping
  27 */
  28
  29#include <linux/pci.h>
  30#include <linux/dmar.h>
  31#include <linux/iova.h>
  32#include <linux/intel-iommu.h>
  33#include <linux/timer.h>
  34#include <linux/irq.h>
  35#include <linux/interrupt.h>
  36#include <linux/tboot.h>
  37#include <linux/dmi.h>
  38#include <linux/slab.h>
  39#include <asm/iommu_table.h>
  40
  41#define PREFIX "DMAR: "
  42
  43/* No locks are needed as DMA remapping hardware unit
  44 * list is constructed at boot time and hotplug of
  45 * these units are not supported by the architecture.
  46 */
  47LIST_HEAD(dmar_drhd_units);
  48
  49static struct acpi_table_header * __initdata dmar_tbl;
  50static acpi_size dmar_tbl_size;
  51
  52static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  53{
  54        /*
  55         * add INCLUDE_ALL at the tail, so scan the list will find it at
  56         * the very end.
  57         */
  58        if (drhd->include_all)
  59                list_add_tail(&drhd->list, &dmar_drhd_units);
  60        else
  61                list_add(&drhd->list, &dmar_drhd_units);
  62}
  63
  64static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  65                                           struct pci_dev **dev, u16 segment)
  66{
  67        struct pci_bus *bus;
  68        struct pci_dev *pdev = NULL;
  69        struct acpi_dmar_pci_path *path;
  70        int count;
  71
  72        bus = pci_find_bus(segment, scope->bus);
  73        path = (struct acpi_dmar_pci_path *)(scope + 1);
  74        count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  75                / sizeof(struct acpi_dmar_pci_path);
  76
  77        while (count) {
  78                if (pdev)
  79                        pci_dev_put(pdev);
  80                /*
  81                 * Some BIOSes list non-exist devices in DMAR table, just
  82                 * ignore it
  83                 */
  84                if (!bus) {
  85                        printk(KERN_WARNING
  86                        PREFIX "Device scope bus [%d] not found\n",
  87                        scope->bus);
  88                        break;
  89                }
  90                pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  91                if (!pdev) {
  92                        printk(KERN_WARNING PREFIX
  93                        "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  94                                segment, bus->number, path->dev, path->fn);
  95                        break;
  96                }
  97                path ++;
  98                count --;
  99                bus = pdev->subordinate;
 100        }
 101        if (!pdev) {
 102                printk(KERN_WARNING PREFIX
 103                "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 104                segment, scope->bus, path->dev, path->fn);
 105                *dev = NULL;
 106                return 0;
 107        }
 108        if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 109                        pdev->subordinate) || (scope->entry_type == \
 110                        ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 111                pci_dev_put(pdev);
 112                printk(KERN_WARNING PREFIX
 113                        "Device scope type does not match for %s\n",
 114                         pci_name(pdev));
 115                return -EINVAL;
 116        }
 117        *dev = pdev;
 118        return 0;
 119}
 120
 121static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 122                                       struct pci_dev ***devices, u16 segment)
 123{
 124        struct acpi_dmar_device_scope *scope;
 125        void * tmp = start;
 126        int index;
 127        int ret;
 128
 129        *cnt = 0;
 130        while (start < end) {
 131                scope = start;
 132                if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 133                    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 134                        (*cnt)++;
 135                else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
 136                        printk(KERN_WARNING PREFIX
 137                               "Unsupported device scope\n");
 138                }
 139                start += scope->length;
 140        }
 141        if (*cnt == 0)
 142                return 0;
 143
 144        *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 145        if (!*devices)
 146                return -ENOMEM;
 147
 148        start = tmp;
 149        index = 0;
 150        while (start < end) {
 151                scope = start;
 152                if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 153                    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 154                        ret = dmar_parse_one_dev_scope(scope,
 155                                &(*devices)[index], segment);
 156                        if (ret) {
 157                                kfree(*devices);
 158                                return ret;
 159                        }
 160                        index ++;
 161                }
 162                start += scope->length;
 163        }
 164
 165        return 0;
 166}
 167
 168/**
 169 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 170 * structure which uniquely represent one DMA remapping hardware unit
 171 * present in the platform
 172 */
 173static int __init
 174dmar_parse_one_drhd(struct acpi_dmar_header *header)
 175{
 176        struct acpi_dmar_hardware_unit *drhd;
 177        struct dmar_drhd_unit *dmaru;
 178        int ret = 0;
 179
 180        drhd = (struct acpi_dmar_hardware_unit *)header;
 181        dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 182        if (!dmaru)
 183                return -ENOMEM;
 184
 185        dmaru->hdr = header;
 186        dmaru->reg_base_addr = drhd->address;
 187        dmaru->segment = drhd->segment;
 188        dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 189
 190        ret = alloc_iommu(dmaru);
 191        if (ret) {
 192                kfree(dmaru);
 193                return ret;
 194        }
 195        dmar_register_drhd_unit(dmaru);
 196        return 0;
 197}
 198
 199static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 200{
 201        struct acpi_dmar_hardware_unit *drhd;
 202        int ret = 0;
 203
 204        drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 205
 206        if (dmaru->include_all)
 207                return 0;
 208
 209        ret = dmar_parse_dev_scope((void *)(drhd + 1),
 210                                ((void *)drhd) + drhd->header.length,
 211                                &dmaru->devices_cnt, &dmaru->devices,
 212                                drhd->segment);
 213        if (ret) {
 214                list_del(&dmaru->list);
 215                kfree(dmaru);
 216        }
 217        return ret;
 218}
 219
 220#ifdef CONFIG_DMAR
 221LIST_HEAD(dmar_rmrr_units);
 222
 223static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 224{
 225        list_add(&rmrr->list, &dmar_rmrr_units);
 226}
 227
 228
 229static int __init
 230dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 231{
 232        struct acpi_dmar_reserved_memory *rmrr;
 233        struct dmar_rmrr_unit *rmrru;
 234
 235        rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 236        if (!rmrru)
 237                return -ENOMEM;
 238
 239        rmrru->hdr = header;
 240        rmrr = (struct acpi_dmar_reserved_memory *)header;
 241        rmrru->base_address = rmrr->base_address;
 242        rmrru->end_address = rmrr->end_address;
 243
 244        dmar_register_rmrr_unit(rmrru);
 245        return 0;
 246}
 247
 248static int __init
 249rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 250{
 251        struct acpi_dmar_reserved_memory *rmrr;
 252        int ret;
 253
 254        rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 255        ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 256                ((void *)rmrr) + rmrr->header.length,
 257                &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 258
 259        if (ret || (rmrru->devices_cnt == 0)) {
 260                list_del(&rmrru->list);
 261                kfree(rmrru);
 262        }
 263        return ret;
 264}
 265
 266static LIST_HEAD(dmar_atsr_units);
 267
 268static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
 269{
 270        struct acpi_dmar_atsr *atsr;
 271        struct dmar_atsr_unit *atsru;
 272
 273        atsr = container_of(hdr, struct acpi_dmar_atsr, header);
 274        atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
 275        if (!atsru)
 276                return -ENOMEM;
 277
 278        atsru->hdr = hdr;
 279        atsru->include_all = atsr->flags & 0x1;
 280
 281        list_add(&atsru->list, &dmar_atsr_units);
 282
 283        return 0;
 284}
 285
 286static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
 287{
 288        int rc;
 289        struct acpi_dmar_atsr *atsr;
 290
 291        if (atsru->include_all)
 292                return 0;
 293
 294        atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 295        rc = dmar_parse_dev_scope((void *)(atsr + 1),
 296                                (void *)atsr + atsr->header.length,
 297                                &atsru->devices_cnt, &atsru->devices,
 298                                atsr->segment);
 299        if (rc || !atsru->devices_cnt) {
 300                list_del(&atsru->list);
 301                kfree(atsru);
 302        }
 303
 304        return rc;
 305}
 306
 307int dmar_find_matched_atsr_unit(struct pci_dev *dev)
 308{
 309        int i;
 310        struct pci_bus *bus;
 311        struct acpi_dmar_atsr *atsr;
 312        struct dmar_atsr_unit *atsru;
 313
 314        dev = pci_physfn(dev);
 315
 316        list_for_each_entry(atsru, &dmar_atsr_units, list) {
 317                atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 318                if (atsr->segment == pci_domain_nr(dev->bus))
 319                        goto found;
 320        }
 321
 322        return 0;
 323
 324found:
 325        for (bus = dev->bus; bus; bus = bus->parent) {
 326                struct pci_dev *bridge = bus->self;
 327
 328                if (!bridge || !pci_is_pcie(bridge) ||
 329                    bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 330                        return 0;
 331
 332                if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
 333                        for (i = 0; i < atsru->devices_cnt; i++)
 334                                if (atsru->devices[i] == bridge)
 335                                        return 1;
 336                        break;
 337                }
 338        }
 339
 340        if (atsru->include_all)
 341                return 1;
 342
 343        return 0;
 344}
 345#endif
 346
 347#ifdef CONFIG_ACPI_NUMA
 348static int __init
 349dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 350{
 351        struct acpi_dmar_rhsa *rhsa;
 352        struct dmar_drhd_unit *drhd;
 353
 354        rhsa = (struct acpi_dmar_rhsa *)header;
 355        for_each_drhd_unit(drhd) {
 356                if (drhd->reg_base_addr == rhsa->base_address) {
 357                        int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 358
 359                        if (!node_online(node))
 360                                node = -1;
 361                        drhd->iommu->node = node;
 362                        return 0;
 363                }
 364        }
 365        WARN_TAINT(
 366                1, TAINT_FIRMWARE_WORKAROUND,
 367                "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 368                "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 369                drhd->reg_base_addr,
 370                dmi_get_system_info(DMI_BIOS_VENDOR),
 371                dmi_get_system_info(DMI_BIOS_VERSION),
 372                dmi_get_system_info(DMI_PRODUCT_VERSION));
 373
 374        return 0;
 375}
 376#endif
 377
 378static void __init
 379dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 380{
 381        struct acpi_dmar_hardware_unit *drhd;
 382        struct acpi_dmar_reserved_memory *rmrr;
 383        struct acpi_dmar_atsr *atsr;
 384        struct acpi_dmar_rhsa *rhsa;
 385
 386        switch (header->type) {
 387        case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 388                drhd = container_of(header, struct acpi_dmar_hardware_unit,
 389                                    header);
 390                printk (KERN_INFO PREFIX
 391                        "DRHD base: %#016Lx flags: %#x\n",
 392                        (unsigned long long)drhd->address, drhd->flags);
 393                break;
 394        case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 395                rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 396                                    header);
 397                printk (KERN_INFO PREFIX
 398                        "RMRR base: %#016Lx end: %#016Lx\n",
 399                        (unsigned long long)rmrr->base_address,
 400                        (unsigned long long)rmrr->end_address);
 401                break;
 402        case ACPI_DMAR_TYPE_ATSR:
 403                atsr = container_of(header, struct acpi_dmar_atsr, header);
 404                printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
 405                break;
 406        case ACPI_DMAR_HARDWARE_AFFINITY:
 407                rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 408                printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
 409                       (unsigned long long)rhsa->base_address,
 410                       rhsa->proximity_domain);
 411                break;
 412        }
 413}
 414
 415/**
 416 * dmar_table_detect - checks to see if the platform supports DMAR devices
 417 */
 418static int __init dmar_table_detect(void)
 419{
 420        acpi_status status = AE_OK;
 421
 422        /* if we could find DMAR table, then there are DMAR devices */
 423        status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 424                                (struct acpi_table_header **)&dmar_tbl,
 425                                &dmar_tbl_size);
 426
 427        if (ACPI_SUCCESS(status) && !dmar_tbl) {
 428                printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 429                status = AE_NOT_FOUND;
 430        }
 431
 432        return (ACPI_SUCCESS(status) ? 1 : 0);
 433}
 434
 435/**
 436 * parse_dmar_table - parses the DMA reporting table
 437 */
 438static int __init
 439parse_dmar_table(void)
 440{
 441        struct acpi_table_dmar *dmar;
 442        struct acpi_dmar_header *entry_header;
 443        int ret = 0;
 444
 445        /*
 446         * Do it again, earlier dmar_tbl mapping could be mapped with
 447         * fixed map.
 448         */
 449        dmar_table_detect();
 450
 451        /*
 452         * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 453         * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 454         */
 455        dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 456
 457        dmar = (struct acpi_table_dmar *)dmar_tbl;
 458        if (!dmar)
 459                return -ENODEV;
 460
 461        if (dmar->width < PAGE_SHIFT - 1) {
 462                printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 463                return -EINVAL;
 464        }
 465
 466        printk (KERN_INFO PREFIX "Host address width %d\n",
 467                dmar->width + 1);
 468
 469        entry_header = (struct acpi_dmar_header *)(dmar + 1);
 470        while (((unsigned long)entry_header) <
 471                        (((unsigned long)dmar) + dmar_tbl->length)) {
 472                /* Avoid looping forever on bad ACPI tables */
 473                if (entry_header->length == 0) {
 474                        printk(KERN_WARNING PREFIX
 475                                "Invalid 0-length structure\n");
 476                        ret = -EINVAL;
 477                        break;
 478                }
 479
 480                dmar_table_print_dmar_entry(entry_header);
 481
 482                switch (entry_header->type) {
 483                case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 484                        ret = dmar_parse_one_drhd(entry_header);
 485                        break;
 486                case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 487#ifdef CONFIG_DMAR
 488                        ret = dmar_parse_one_rmrr(entry_header);
 489#endif
 490                        break;
 491                case ACPI_DMAR_TYPE_ATSR:
 492#ifdef CONFIG_DMAR
 493                        ret = dmar_parse_one_atsr(entry_header);
 494#endif
 495                        break;
 496                case ACPI_DMAR_HARDWARE_AFFINITY:
 497#ifdef CONFIG_ACPI_NUMA
 498                        ret = dmar_parse_one_rhsa(entry_header);
 499#endif
 500                        break;
 501                default:
 502                        printk(KERN_WARNING PREFIX
 503                                "Unknown DMAR structure type %d\n",
 504                                entry_header->type);
 505                        ret = 0; /* for forward compatibility */
 506                        break;
 507                }
 508                if (ret)
 509                        break;
 510
 511                entry_header = ((void *)entry_header + entry_header->length);
 512        }
 513        return ret;
 514}
 515
 516static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 517                          struct pci_dev *dev)
 518{
 519        int index;
 520
 521        while (dev) {
 522                for (index = 0; index < cnt; index++)
 523                        if (dev == devices[index])
 524                                return 1;
 525
 526                /* Check our parent */
 527                dev = dev->bus->self;
 528        }
 529
 530        return 0;
 531}
 532
 533struct dmar_drhd_unit *
 534dmar_find_matched_drhd_unit(struct pci_dev *dev)
 535{
 536        struct dmar_drhd_unit *dmaru = NULL;
 537        struct acpi_dmar_hardware_unit *drhd;
 538
 539        dev = pci_physfn(dev);
 540
 541        list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 542                drhd = container_of(dmaru->hdr,
 543                                    struct acpi_dmar_hardware_unit,
 544                                    header);
 545
 546                if (dmaru->include_all &&
 547                    drhd->segment == pci_domain_nr(dev->bus))
 548                        return dmaru;
 549
 550                if (dmar_pci_device_match(dmaru->devices,
 551                                          dmaru->devices_cnt, dev))
 552                        return dmaru;
 553        }
 554
 555        return NULL;
 556}
 557
 558int __init dmar_dev_scope_init(void)
 559{
 560        struct dmar_drhd_unit *drhd, *drhd_n;
 561        int ret = -ENODEV;
 562
 563        list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 564                ret = dmar_parse_dev(drhd);
 565                if (ret)
 566                        return ret;
 567        }
 568
 569#ifdef CONFIG_DMAR
 570        {
 571                struct dmar_rmrr_unit *rmrr, *rmrr_n;
 572                struct dmar_atsr_unit *atsr, *atsr_n;
 573
 574                list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 575                        ret = rmrr_parse_dev(rmrr);
 576                        if (ret)
 577                                return ret;
 578                }
 579
 580                list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
 581                        ret = atsr_parse_dev(atsr);
 582                        if (ret)
 583                                return ret;
 584                }
 585        }
 586#endif
 587
 588        return ret;
 589}
 590
 591
 592int __init dmar_table_init(void)
 593{
 594        static int dmar_table_initialized;
 595        int ret;
 596
 597        if (dmar_table_initialized)
 598                return 0;
 599
 600        dmar_table_initialized = 1;
 601
 602        ret = parse_dmar_table();
 603        if (ret) {
 604                if (ret != -ENODEV)
 605                        printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 606                return ret;
 607        }
 608
 609        if (list_empty(&dmar_drhd_units)) {
 610                printk(KERN_INFO PREFIX "No DMAR devices found\n");
 611                return -ENODEV;
 612        }
 613
 614#ifdef CONFIG_DMAR
 615        if (list_empty(&dmar_rmrr_units))
 616                printk(KERN_INFO PREFIX "No RMRR found\n");
 617
 618        if (list_empty(&dmar_atsr_units))
 619                printk(KERN_INFO PREFIX "No ATSR found\n");
 620#endif
 621
 622        return 0;
 623}
 624
 625static void warn_invalid_dmar(u64 addr, const char *message)
 626{
 627        WARN_TAINT_ONCE(
 628                1, TAINT_FIRMWARE_WORKAROUND,
 629                "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 630                "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 631                addr, message,
 632                dmi_get_system_info(DMI_BIOS_VENDOR),
 633                dmi_get_system_info(DMI_BIOS_VERSION),
 634                dmi_get_system_info(DMI_PRODUCT_VERSION));
 635}
 636
 637int __init check_zero_address(void)
 638{
 639        struct acpi_table_dmar *dmar;
 640        struct acpi_dmar_header *entry_header;
 641        struct acpi_dmar_hardware_unit *drhd;
 642
 643        dmar = (struct acpi_table_dmar *)dmar_tbl;
 644        entry_header = (struct acpi_dmar_header *)(dmar + 1);
 645
 646        while (((unsigned long)entry_header) <
 647                        (((unsigned long)dmar) + dmar_tbl->length)) {
 648                /* Avoid looping forever on bad ACPI tables */
 649                if (entry_header->length == 0) {
 650                        printk(KERN_WARNING PREFIX
 651                                "Invalid 0-length structure\n");
 652                        return 0;
 653                }
 654
 655                if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 656                        void __iomem *addr;
 657                        u64 cap, ecap;
 658
 659                        drhd = (void *)entry_header;
 660                        if (!drhd->address) {
 661                                warn_invalid_dmar(0, "");
 662                                goto failed;
 663                        }
 664
 665                        addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 666                        if (!addr ) {
 667                                printk("IOMMU: can't validate: %llx\n", drhd->address);
 668                                goto failed;
 669                        }
 670                        cap = dmar_readq(addr + DMAR_CAP_REG);
 671                        ecap = dmar_readq(addr + DMAR_ECAP_REG);
 672                        early_iounmap(addr, VTD_PAGE_SIZE);
 673                        if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 674                                warn_invalid_dmar(drhd->address,
 675                                                  " returns all ones");
 676                                goto failed;
 677                        }
 678                }
 679
 680                entry_header = ((void *)entry_header + entry_header->length);
 681        }
 682        return 1;
 683
 684failed:
 685#ifdef CONFIG_DMAR
 686        dmar_disabled = 1;
 687#endif
 688        return 0;
 689}
 690
 691int __init detect_intel_iommu(void)
 692{
 693        int ret;
 694
 695        ret = dmar_table_detect();
 696        if (ret)
 697                ret = check_zero_address();
 698        {
 699#ifdef CONFIG_INTR_REMAP
 700                struct acpi_table_dmar *dmar;
 701                /*
 702                 * for now we will disable dma-remapping when interrupt
 703                 * remapping is enabled.
 704                 * When support for queued invalidation for IOTLB invalidation
 705                 * is added, we will not need this any more.
 706                 */
 707                dmar = (struct acpi_table_dmar *) dmar_tbl;
 708                if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 709                        printk(KERN_INFO
 710                               "Queued invalidation will be enabled to support "
 711                               "x2apic and Intr-remapping.\n");
 712#endif
 713#ifdef CONFIG_DMAR
 714                if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 715                        iommu_detected = 1;
 716                        /* Make sure ACS will be enabled */
 717                        pci_request_acs();
 718                }
 719#endif
 720#ifdef CONFIG_X86
 721                if (ret)
 722                        x86_init.iommu.iommu_init = intel_iommu_init;
 723#endif
 724        }
 725        early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 726        dmar_tbl = NULL;
 727
 728        return ret ? 1 : -ENODEV;
 729}
 730
 731
 732int alloc_iommu(struct dmar_drhd_unit *drhd)
 733{
 734        struct intel_iommu *iommu;
 735        int map_size;
 736        u32 ver;
 737        static int iommu_allocated = 0;
 738        int agaw = 0;
 739        int msagaw = 0;
 740
 741        if (!drhd->reg_base_addr) {
 742                warn_invalid_dmar(0, "");
 743                return -EINVAL;
 744        }
 745
 746        iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 747        if (!iommu)
 748                return -ENOMEM;
 749
 750        iommu->seq_id = iommu_allocated++;
 751        sprintf (iommu->name, "dmar%d", iommu->seq_id);
 752
 753        iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 754        if (!iommu->reg) {
 755                printk(KERN_ERR "IOMMU: can't map the region\n");
 756                goto error;
 757        }
 758        iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 759        iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 760
 761        if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 762                warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
 763                goto err_unmap;
 764        }
 765
 766#ifdef CONFIG_DMAR
 767        agaw = iommu_calculate_agaw(iommu);
 768        if (agaw < 0) {
 769                printk(KERN_ERR
 770                       "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 771                       iommu->seq_id);
 772                goto err_unmap;
 773        }
 774        msagaw = iommu_calculate_max_sagaw(iommu);
 775        if (msagaw < 0) {
 776                printk(KERN_ERR
 777                        "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 778                        iommu->seq_id);
 779                goto err_unmap;
 780        }
 781#endif
 782        iommu->agaw = agaw;
 783        iommu->msagaw = msagaw;
 784
 785        iommu->node = -1;
 786
 787        /* the registers might be more than one page */
 788        map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 789                cap_max_fault_reg_offset(iommu->cap));
 790        map_size = VTD_PAGE_ALIGN(map_size);
 791        if (map_size > VTD_PAGE_SIZE) {
 792                iounmap(iommu->reg);
 793                iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 794                if (!iommu->reg) {
 795                        printk(KERN_ERR "IOMMU: can't map the region\n");
 796                        goto error;
 797                }
 798        }
 799
 800        ver = readl(iommu->reg + DMAR_VER_REG);
 801        pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 802                iommu->seq_id,
 803                (unsigned long long)drhd->reg_base_addr,
 804                DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 805                (unsigned long long)iommu->cap,
 806                (unsigned long long)iommu->ecap);
 807
 808        spin_lock_init(&iommu->register_lock);
 809
 810        drhd->iommu = iommu;
 811        return 0;
 812
 813 err_unmap:
 814        iounmap(iommu->reg);
 815 error:
 816        kfree(iommu);
 817        return -1;
 818}
 819
 820void free_iommu(struct intel_iommu *iommu)
 821{
 822        if (!iommu)
 823                return;
 824
 825#ifdef CONFIG_DMAR
 826        free_dmar_iommu(iommu);
 827#endif
 828
 829        if (iommu->reg)
 830                iounmap(iommu->reg);
 831        kfree(iommu);
 832}
 833
 834/*
 835 * Reclaim all the submitted descriptors which have completed its work.
 836 */
 837static inline void reclaim_free_desc(struct q_inval *qi)
 838{
 839        while (qi->desc_status[qi->free_tail] == QI_DONE ||
 840               qi->desc_status[qi->free_tail] == QI_ABORT) {
 841                qi->desc_status[qi->free_tail] = QI_FREE;
 842                qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 843                qi->free_cnt++;
 844        }
 845}
 846
 847static int qi_check_fault(struct intel_iommu *iommu, int index)
 848{
 849        u32 fault;
 850        int head, tail;
 851        struct q_inval *qi = iommu->qi;
 852        int wait_index = (index + 1) % QI_LENGTH;
 853
 854        if (qi->desc_status[wait_index] == QI_ABORT)
 855                return -EAGAIN;
 856
 857        fault = readl(iommu->reg + DMAR_FSTS_REG);
 858
 859        /*
 860         * If IQE happens, the head points to the descriptor associated
 861         * with the error. No new descriptors are fetched until the IQE
 862         * is cleared.
 863         */
 864        if (fault & DMA_FSTS_IQE) {
 865                head = readl(iommu->reg + DMAR_IQH_REG);
 866                if ((head >> DMAR_IQ_SHIFT) == index) {
 867                        printk(KERN_ERR "VT-d detected invalid descriptor: "
 868                                "low=%llx, high=%llx\n",
 869                                (unsigned long long)qi->desc[index].low,
 870                                (unsigned long long)qi->desc[index].high);
 871                        memcpy(&qi->desc[index], &qi->desc[wait_index],
 872                                        sizeof(struct qi_desc));
 873                        __iommu_flush_cache(iommu, &qi->desc[index],
 874                                        sizeof(struct qi_desc));
 875                        writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 876                        return -EINVAL;
 877                }
 878        }
 879
 880        /*
 881         * If ITE happens, all pending wait_desc commands are aborted.
 882         * No new descriptors are fetched until the ITE is cleared.
 883         */
 884        if (fault & DMA_FSTS_ITE) {
 885                head = readl(iommu->reg + DMAR_IQH_REG);
 886                head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 887                head |= 1;
 888                tail = readl(iommu->reg + DMAR_IQT_REG);
 889                tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 890
 891                writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 892
 893                do {
 894                        if (qi->desc_status[head] == QI_IN_USE)
 895                                qi->desc_status[head] = QI_ABORT;
 896                        head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 897                } while (head != tail);
 898
 899                if (qi->desc_status[wait_index] == QI_ABORT)
 900                        return -EAGAIN;
 901        }
 902
 903        if (fault & DMA_FSTS_ICE)
 904                writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 905
 906        return 0;
 907}
 908
 909/*
 910 * Submit the queued invalidation descriptor to the remapping
 911 * hardware unit and wait for its completion.
 912 */
 913int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 914{
 915        int rc;
 916        struct q_inval *qi = iommu->qi;
 917        struct qi_desc *hw, wait_desc;
 918        int wait_index, index;
 919        unsigned long flags;
 920
 921        if (!qi)
 922                return 0;
 923
 924        hw = qi->desc;
 925
 926restart:
 927        rc = 0;
 928
 929        spin_lock_irqsave(&qi->q_lock, flags);
 930        while (qi->free_cnt < 3) {
 931                spin_unlock_irqrestore(&qi->q_lock, flags);
 932                cpu_relax();
 933                spin_lock_irqsave(&qi->q_lock, flags);
 934        }
 935
 936        index = qi->free_head;
 937        wait_index = (index + 1) % QI_LENGTH;
 938
 939        qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 940
 941        hw[index] = *desc;
 942
 943        wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 944                        QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 945        wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 946
 947        hw[wait_index] = wait_desc;
 948
 949        __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 950        __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 951
 952        qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 953        qi->free_cnt -= 2;
 954
 955        /*
 956         * update the HW tail register indicating the presence of
 957         * new descriptors.
 958         */
 959        writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 960
 961        while (qi->desc_status[wait_index] != QI_DONE) {
 962                /*
 963                 * We will leave the interrupts disabled, to prevent interrupt
 964                 * context to queue another cmd while a cmd is already submitted
 965                 * and waiting for completion on this cpu. This is to avoid
 966                 * a deadlock where the interrupt context can wait indefinitely
 967                 * for free slots in the queue.
 968                 */
 969                rc = qi_check_fault(iommu, index);
 970                if (rc)
 971                        break;
 972
 973                spin_unlock(&qi->q_lock);
 974                cpu_relax();
 975                spin_lock(&qi->q_lock);
 976        }
 977
 978        qi->desc_status[index] = QI_DONE;
 979
 980        reclaim_free_desc(qi);
 981        spin_unlock_irqrestore(&qi->q_lock, flags);
 982
 983        if (rc == -EAGAIN)
 984                goto restart;
 985
 986        return rc;
 987}
 988
 989/*
 990 * Flush the global interrupt entry cache.
 991 */
 992void qi_global_iec(struct intel_iommu *iommu)
 993{
 994        struct qi_desc desc;
 995
 996        desc.low = QI_IEC_TYPE;
 997        desc.high = 0;
 998
 999        /* should never fail */
1000        qi_submit_sync(&desc, iommu);
1001}
1002
1003void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1004                      u64 type)
1005{
1006        struct qi_desc desc;
1007
1008        desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1009                        | QI_CC_GRAN(type) | QI_CC_TYPE;
1010        desc.high = 0;
1011
1012        qi_submit_sync(&desc, iommu);
1013}
1014
1015void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1016                    unsigned int size_order, u64 type)
1017{
1018        u8 dw = 0, dr = 0;
1019
1020        struct qi_desc desc;
1021        int ih = 0;
1022
1023        if (cap_write_drain(iommu->cap))
1024                dw = 1;
1025
1026        if (cap_read_drain(iommu->cap))
1027                dr = 1;
1028
1029        desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1030                | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1031        desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1032                | QI_IOTLB_AM(size_order);
1033
1034        qi_submit_sync(&desc, iommu);
1035}
1036
1037void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1038                        u64 addr, unsigned mask)
1039{
1040        struct qi_desc desc;
1041
1042        if (mask) {
1043                BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1044                addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1045                desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1046        } else
1047                desc.high = QI_DEV_IOTLB_ADDR(addr);
1048
1049        if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1050                qdep = 0;
1051
1052        desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1053                   QI_DIOTLB_TYPE;
1054
1055        qi_submit_sync(&desc, iommu);
1056}
1057
1058/*
1059 * Disable Queued Invalidation interface.
1060 */
1061void dmar_disable_qi(struct intel_iommu *iommu)
1062{
1063        unsigned long flags;
1064        u32 sts;
1065        cycles_t start_time = get_cycles();
1066
1067        if (!ecap_qis(iommu->ecap))
1068                return;
1069
1070        spin_lock_irqsave(&iommu->register_lock, flags);
1071
1072        sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
1073        if (!(sts & DMA_GSTS_QIES))
1074                goto end;
1075
1076        /*
1077         * Give a chance to HW to complete the pending invalidation requests.
1078         */
1079        while ((readl(iommu->reg + DMAR_IQT_REG) !=
1080                readl(iommu->reg + DMAR_IQH_REG)) &&
1081                (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1082                cpu_relax();
1083
1084        iommu->gcmd &= ~DMA_GCMD_QIE;
1085        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1086
1087        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1088                      !(sts & DMA_GSTS_QIES), sts);
1089end:
1090        spin_unlock_irqrestore(&iommu->register_lock, flags);
1091}
1092
1093/*
1094 * Enable queued invalidation.
1095 */
1096static void __dmar_enable_qi(struct intel_iommu *iommu)
1097{
1098        u32 sts;
1099        unsigned long flags;
1100        struct q_inval *qi = iommu->qi;
1101
1102        qi->free_head = qi->free_tail = 0;
1103        qi->free_cnt = QI_LENGTH;
1104
1105        spin_lock_irqsave(&iommu->register_lock, flags);
1106
1107        /* write zero to the tail reg */
1108        writel(0, iommu->reg + DMAR_IQT_REG);
1109
1110        dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1111
1112        iommu->gcmd |= DMA_GCMD_QIE;
1113        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1114
1115        /* Make sure hardware complete it */
1116        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1117
1118        spin_unlock_irqrestore(&iommu->register_lock, flags);
1119}
1120
1121/*
1122 * Enable Queued Invalidation interface. This is a must to support
1123 * interrupt-remapping. Also used by DMA-remapping, which replaces
1124 * register based IOTLB invalidation.
1125 */
1126int dmar_enable_qi(struct intel_iommu *iommu)
1127{
1128        struct q_inval *qi;
1129        struct page *desc_page;
1130
1131        if (!ecap_qis(iommu->ecap))
1132                return -ENOENT;
1133
1134        /*
1135         * queued invalidation is already setup and enabled.
1136         */
1137        if (iommu->qi)
1138                return 0;
1139
1140        iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1141        if (!iommu->qi)
1142                return -ENOMEM;
1143
1144        qi = iommu->qi;
1145
1146
1147        desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1148        if (!desc_page) {
1149                kfree(qi);
1150                iommu->qi = 0;
1151                return -ENOMEM;
1152        }
1153
1154        qi->desc = page_address(desc_page);
1155
1156        qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1157        if (!qi->desc_status) {
1158                free_page((unsigned long) qi->desc);
1159                kfree(qi);
1160                iommu->qi = 0;
1161                return -ENOMEM;
1162        }
1163
1164        qi->free_head = qi->free_tail = 0;
1165        qi->free_cnt = QI_LENGTH;
1166
1167        spin_lock_init(&qi->q_lock);
1168
1169        __dmar_enable_qi(iommu);
1170
1171        return 0;
1172}
1173
1174/* iommu interrupt handling. Most stuff are MSI-like. */
1175
1176enum faulttype {
1177        DMA_REMAP,
1178        INTR_REMAP,
1179        UNKNOWN,
1180};
1181
1182static const char *dma_remap_fault_reasons[] =
1183{
1184        "Software",
1185        "Present bit in root entry is clear",
1186        "Present bit in context entry is clear",
1187        "Invalid context entry",
1188        "Access beyond MGAW",
1189        "PTE Write access is not set",
1190        "PTE Read access is not set",
1191        "Next page table ptr is invalid",
1192        "Root table address invalid",
1193        "Context table ptr is invalid",
1194        "non-zero reserved fields in RTP",
1195        "non-zero reserved fields in CTP",
1196        "non-zero reserved fields in PTE",
1197};
1198
1199static const char *intr_remap_fault_reasons[] =
1200{
1201        "Detected reserved fields in the decoded interrupt-remapped request",
1202        "Interrupt index exceeded the interrupt-remapping table size",
1203        "Present field in the IRTE entry is clear",
1204        "Error accessing interrupt-remapping table pointed by IRTA_REG",
1205        "Detected reserved fields in the IRTE entry",
1206        "Blocked a compatibility format interrupt request",
1207        "Blocked an interrupt request due to source-id verification failure",
1208};
1209
1210#define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1211
1212const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1213{
1214        if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1215                                     ARRAY_SIZE(intr_remap_fault_reasons))) {
1216                *fault_type = INTR_REMAP;
1217                return intr_remap_fault_reasons[fault_reason - 0x20];
1218        } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1219                *fault_type = DMA_REMAP;
1220                return dma_remap_fault_reasons[fault_reason];
1221        } else {
1222                *fault_type = UNKNOWN;
1223                return "Unknown";
1224        }
1225}
1226
1227void dmar_msi_unmask(struct irq_data *data)
1228{
1229        struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1230        unsigned long flag;
1231
1232        /* unmask it */
1233        spin_lock_irqsave(&iommu->register_lock, flag);
1234        writel(0, iommu->reg + DMAR_FECTL_REG);
1235        /* Read a reg to force flush the post write */
1236        readl(iommu->reg + DMAR_FECTL_REG);
1237        spin_unlock_irqrestore(&iommu->register_lock, flag);
1238}
1239
1240void dmar_msi_mask(struct irq_data *data)
1241{
1242        unsigned long flag;
1243        struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1244
1245        /* mask it */
1246        spin_lock_irqsave(&iommu->register_lock, flag);
1247        writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1248        /* Read a reg to force flush the post write */
1249        readl(iommu->reg + DMAR_FECTL_REG);
1250        spin_unlock_irqrestore(&iommu->register_lock, flag);
1251}
1252
1253void dmar_msi_write(int irq, struct msi_msg *msg)
1254{
1255        struct intel_iommu *iommu = irq_get_handler_data(irq);
1256        unsigned long flag;
1257
1258        spin_lock_irqsave(&iommu->register_lock, flag);
1259        writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1260        writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1261        writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1262        spin_unlock_irqrestore(&iommu->register_lock, flag);
1263}
1264
1265void dmar_msi_read(int irq, struct msi_msg *msg)
1266{
1267        struct intel_iommu *iommu = irq_get_handler_data(irq);
1268        unsigned long flag;
1269
1270        spin_lock_irqsave(&iommu->register_lock, flag);
1271        msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1272        msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1273        msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1274        spin_unlock_irqrestore(&iommu->register_lock, flag);
1275}
1276
1277static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1278                u8 fault_reason, u16 source_id, unsigned long long addr)
1279{
1280        const char *reason;
1281        int fault_type;
1282
1283        reason = dmar_get_fault_reason(fault_reason, &fault_type);
1284
1285        if (fault_type == INTR_REMAP)
1286                printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1287                       "fault index %llx\n"
1288                        "INTR-REMAP:[fault reason %02d] %s\n",
1289                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1290                        PCI_FUNC(source_id & 0xFF), addr >> 48,
1291                        fault_reason, reason);
1292        else
1293                printk(KERN_ERR
1294                       "DMAR:[%s] Request device [%02x:%02x.%d] "
1295                       "fault addr %llx \n"
1296                       "DMAR:[fault reason %02d] %s\n",
1297                       (type ? "DMA Read" : "DMA Write"),
1298                       (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1299                       PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1300        return 0;
1301}
1302
1303#define PRIMARY_FAULT_REG_LEN (16)
1304irqreturn_t dmar_fault(int irq, void *dev_id)
1305{
1306        struct intel_iommu *iommu = dev_id;
1307        int reg, fault_index;
1308        u32 fault_status;
1309        unsigned long flag;
1310
1311        spin_lock_irqsave(&iommu->register_lock, flag);
1312        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1313        if (fault_status)
1314                printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1315                       fault_status);
1316
1317        /* TBD: ignore advanced fault log currently */
1318        if (!(fault_status & DMA_FSTS_PPF))
1319                goto clear_rest;
1320
1321        fault_index = dma_fsts_fault_record_index(fault_status);
1322        reg = cap_fault_reg_offset(iommu->cap);
1323        while (1) {
1324                u8 fault_reason;
1325                u16 source_id;
1326                u64 guest_addr;
1327                int type;
1328                u32 data;
1329
1330                /* highest 32 bits */
1331                data = readl(iommu->reg + reg +
1332                                fault_index * PRIMARY_FAULT_REG_LEN + 12);
1333                if (!(data & DMA_FRCD_F))
1334                        break;
1335
1336                fault_reason = dma_frcd_fault_reason(data);
1337                type = dma_frcd_type(data);
1338
1339                data = readl(iommu->reg + reg +
1340                                fault_index * PRIMARY_FAULT_REG_LEN + 8);
1341                source_id = dma_frcd_source_id(data);
1342
1343                guest_addr = dmar_readq(iommu->reg + reg +
1344                                fault_index * PRIMARY_FAULT_REG_LEN);
1345                guest_addr = dma_frcd_page_addr(guest_addr);
1346                /* clear the fault */
1347                writel(DMA_FRCD_F, iommu->reg + reg +
1348                        fault_index * PRIMARY_FAULT_REG_LEN + 12);
1349
1350                spin_unlock_irqrestore(&iommu->register_lock, flag);
1351
1352                dmar_fault_do_one(iommu, type, fault_reason,
1353                                source_id, guest_addr);
1354
1355                fault_index++;
1356                if (fault_index >= cap_num_fault_regs(iommu->cap))
1357                        fault_index = 0;
1358                spin_lock_irqsave(&iommu->register_lock, flag);
1359        }
1360clear_rest:
1361        /* clear all the other faults */
1362        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1363        writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1364
1365        spin_unlock_irqrestore(&iommu->register_lock, flag);
1366        return IRQ_HANDLED;
1367}
1368
1369int dmar_set_interrupt(struct intel_iommu *iommu)
1370{
1371        int irq, ret;
1372
1373        /*
1374         * Check if the fault interrupt is already initialized.
1375         */
1376        if (iommu->irq)
1377                return 0;
1378
1379        irq = create_irq();
1380        if (!irq) {
1381                printk(KERN_ERR "IOMMU: no free vectors\n");
1382                return -EINVAL;
1383        }
1384
1385        irq_set_handler_data(irq, iommu);
1386        iommu->irq = irq;
1387
1388        ret = arch_setup_dmar_msi(irq);
1389        if (ret) {
1390                irq_set_handler_data(irq, NULL);
1391                iommu->irq = 0;
1392                destroy_irq(irq);
1393                return ret;
1394        }
1395
1396        ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1397        if (ret)
1398                printk(KERN_ERR "IOMMU: can't request irq\n");
1399        return ret;
1400}
1401
1402int __init enable_drhd_fault_handling(void)
1403{
1404        struct dmar_drhd_unit *drhd;
1405
1406        /*
1407         * Enable fault control interrupt.
1408         */
1409        for_each_drhd_unit(drhd) {
1410                int ret;
1411                struct intel_iommu *iommu = drhd->iommu;
1412                ret = dmar_set_interrupt(iommu);
1413
1414                if (ret) {
1415                        printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1416                               " interrupt, ret %d\n",
1417                               (unsigned long long)drhd->reg_base_addr, ret);
1418                        return -1;
1419                }
1420
1421                /*
1422                 * Clear any previous faults.
1423                 */
1424                dmar_fault(iommu->irq, iommu);
1425        }
1426
1427        return 0;
1428}
1429
1430/*
1431 * Re-enable Queued Invalidation interface.
1432 */
1433int dmar_reenable_qi(struct intel_iommu *iommu)
1434{
1435        if (!ecap_qis(iommu->ecap))
1436                return -ENOENT;
1437
1438        if (!iommu->qi)
1439                return -ENOENT;
1440
1441        /*
1442         * First disable queued invalidation.
1443         */
1444        dmar_disable_qi(iommu);
1445        /*
1446         * Then enable queued invalidation again. Since there is no pending
1447         * invalidation requests now, it's safe to re-enable queued
1448         * invalidation.
1449         */
1450        __dmar_enable_qi(iommu);
1451
1452        return 0;
1453}
1454
1455/*
1456 * Check interrupt remapping support in DMAR table description.
1457 */
1458int __init dmar_ir_support(void)
1459{
1460        struct acpi_table_dmar *dmar;
1461        dmar = (struct acpi_table_dmar *)dmar_tbl;
1462        if (!dmar)
1463                return 0;
1464        return dmar->flags & 0x1;
1465}
1466IOMMU_INIT_POST(detect_intel_iommu);
1467
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.