LXR linux/drivers/dma/ioat

   1/*
   2 * Intel I/OAT DMA Linux driver
   3 * Copyright(c) 2007 Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope that it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 * You should have received a copy of the GNU General Public License along with
  15 * this program; if not, write to the Free Software Foundation, Inc.,
  16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17 *
  18 * The full GNU General Public License is included in this distribution in
  19 * the file called "COPYING".
  20 *
  21 */
  22
  23#include <linux/kernel.h>
  24#include <linux/pci.h>
  25#include <linux/smp.h>
  26#include <linux/interrupt.h>
  27#include <linux/dca.h>
  28
  29/* either a kernel change is needed, or we need something like this in kernel */
  30#ifndef CONFIG_SMP
  31#include <asm/smp.h>
  32#undef cpu_physical_id
  33#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
  34#endif
  35
  36#include "ioatdma.h"
  37#include "ioatdma_registers.h"
  38
  39/*
  40 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
  41 * contain the bit number of the APIC ID to map into the DCA tag.  If the valid
  42 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
  43 */
  44#define DCA_TAG_MAP_VALID 0x80
  45
  46#define DCA3_TAG_MAP_BIT_TO_INV 0x80
  47#define DCA3_TAG_MAP_BIT_TO_SEL 0x40
  48#define DCA3_TAG_MAP_LITERAL_VAL 0x1
  49
  50#define DCA_TAG_MAP_MASK 0xDF
  51
  52/*
  53 * "Legacy" DCA systems do not implement the DCA register set in the
  54 * I/OAT device.  Software needs direct support for their tag mappings.
  55 */
  56
  57#define APICID_BIT(x)           (DCA_TAG_MAP_VALID | (x))
  58#define IOAT_TAG_MAP_LEN        8
  59
  60static u8 ioat_tag_map_BNB[IOAT_TAG_MAP_LEN] = {
  61        1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
  62static u8 ioat_tag_map_SCNB[IOAT_TAG_MAP_LEN] = {
  63        1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
  64static u8 ioat_tag_map_CNB[IOAT_TAG_MAP_LEN] = {
  65        1, APICID_BIT(1), APICID_BIT(3), APICID_BIT(4), APICID_BIT(2), };
  66static u8 ioat_tag_map_UNISYS[IOAT_TAG_MAP_LEN] = { 0 };
  67
  68/* pack PCI B/D/F into a u16 */
  69static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
  70{
  71        return (pci->bus->number << 8) | pci->devfn;
  72}
  73
  74static int dca_enabled_in_bios(struct pci_dev *pdev)
  75{
  76        /* CPUID level 9 returns DCA configuration */
  77        /* Bit 0 indicates DCA enabled by the BIOS */
  78        unsigned long cpuid_level_9;
  79        int res;
  80
  81        cpuid_level_9 = cpuid_eax(9);
  82        res = test_bit(0, &cpuid_level_9);
  83        if (!res)
  84                dev_err(&pdev->dev, "DCA is disabled in BIOS\n");
  85
  86        return res;
  87}
  88
  89static int system_has_dca_enabled(struct pci_dev *pdev)
  90{
  91        if (boot_cpu_has(X86_FEATURE_DCA))
  92                return dca_enabled_in_bios(pdev);
  93
  94        dev_err(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
  95        return 0;
  96}
  97
  98struct ioat_dca_slot {
  99        struct pci_dev *pdev;   /* requester device */
 100        u16 rid;                /* requester id, as used by IOAT */
 101};
 102
 103#define IOAT_DCA_MAX_REQ 6
 104#define IOAT3_DCA_MAX_REQ 2
 105
 106struct ioat_dca_priv {
 107        void __iomem            *iobase;
 108        void __iomem            *dca_base;
 109        int                      max_requesters;
 110        int                      requester_count;
 111        u8                       tag_map[IOAT_TAG_MAP_LEN];
 112        struct ioat_dca_slot     req_slots[0];
 113};
 114
 115/* 5000 series chipset DCA Port Requester ID Table Entry Format
 116 * [15:8]       PCI-Express Bus Number
 117 * [7:3]        PCI-Express Device Number
 118 * [2:0]        PCI-Express Function Number
 119 *
 120 * 5000 series chipset DCA control register format
 121 * [7:1]        Reserved (0)
 122 * [0]          Ignore Function Number
 123 */
 124
 125static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
 126{
 127        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 128        struct pci_dev *pdev;
 129        int i;
 130        u16 id;
 131
 132        /* This implementation only supports PCI-Express */
 133        if (dev->bus != &pci_bus_type)
 134                return -ENODEV;
 135        pdev = to_pci_dev(dev);
 136        id = dcaid_from_pcidev(pdev);
 137
 138        if (ioatdca->requester_count == ioatdca->max_requesters)
 139                return -ENODEV;
 140
 141        for (i = 0; i < ioatdca->max_requesters; i++) {
 142                if (ioatdca->req_slots[i].pdev == NULL) {
 143                        /* found an empty slot */
 144                        ioatdca->requester_count++;
 145                        ioatdca->req_slots[i].pdev = pdev;
 146                        ioatdca->req_slots[i].rid = id;
 147                        writew(id, ioatdca->dca_base + (i * 4));
 148                        /* make sure the ignore function bit is off */
 149                        writeb(0, ioatdca->dca_base + (i * 4) + 2);
 150                        return i;
 151                }
 152        }
 153        /* Error, ioatdma->requester_count is out of whack */
 154        return -EFAULT;
 155}
 156
 157static int ioat_dca_remove_requester(struct dca_provider *dca,
 158                                     struct device *dev)
 159{
 160        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 161        struct pci_dev *pdev;
 162        int i;
 163
 164        /* This implementation only supports PCI-Express */
 165        if (dev->bus != &pci_bus_type)
 166                return -ENODEV;
 167        pdev = to_pci_dev(dev);
 168
 169        for (i = 0; i < ioatdca->max_requesters; i++) {
 170                if (ioatdca->req_slots[i].pdev == pdev) {
 171                        writew(0, ioatdca->dca_base + (i * 4));
 172                        ioatdca->req_slots[i].pdev = NULL;
 173                        ioatdca->req_slots[i].rid = 0;
 174                        ioatdca->requester_count--;
 175                        return i;
 176                }
 177        }
 178        return -ENODEV;
 179}
 180
 181static u8 ioat_dca_get_tag(struct dca_provider *dca,
 182                           struct device *dev,
 183                           int cpu)
 184{
 185        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 186        int i, apic_id, bit, value;
 187        u8 entry, tag;
 188
 189        tag = 0;
 190        apic_id = cpu_physical_id(cpu);
 191
 192        for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
 193                entry = ioatdca->tag_map[i];
 194                if (entry & DCA_TAG_MAP_VALID) {
 195                        bit = entry & ~DCA_TAG_MAP_VALID;
 196                        value = (apic_id & (1 << bit)) ? 1 : 0;
 197                } else {
 198                        value = entry ? 1 : 0;
 199                }
 200                tag |= (value << i);
 201        }
 202        return tag;
 203}
 204
 205static int ioat_dca_dev_managed(struct dca_provider *dca,
 206                                struct device *dev)
 207{
 208        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 209        struct pci_dev *pdev;
 210        int i;
 211
 212        pdev = to_pci_dev(dev);
 213        for (i = 0; i < ioatdca->max_requesters; i++) {
 214                if (ioatdca->req_slots[i].pdev == pdev)
 215                        return 1;
 216        }
 217        return 0;
 218}
 219
 220static struct dca_ops ioat_dca_ops = {
 221        .add_requester          = ioat_dca_add_requester,
 222        .remove_requester       = ioat_dca_remove_requester,
 223        .get_tag                = ioat_dca_get_tag,
 224        .dev_managed            = ioat_dca_dev_managed,
 225};
 226
 227
 228struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 229{
 230        struct dca_provider *dca;
 231        struct ioat_dca_priv *ioatdca;
 232        u8 *tag_map = NULL;
 233        int i;
 234        int err;
 235        u8 version;
 236        u8 max_requesters;
 237
 238        if (!system_has_dca_enabled(pdev))
 239                return NULL;
 240
 241        /* I/OAT v1 systems must have a known tag_map to support DCA */
 242        switch (pdev->vendor) {
 243        case PCI_VENDOR_ID_INTEL:
 244                switch (pdev->device) {
 245                case PCI_DEVICE_ID_INTEL_IOAT:
 246                        tag_map = ioat_tag_map_BNB;
 247                        break;
 248                case PCI_DEVICE_ID_INTEL_IOAT_CNB:
 249                        tag_map = ioat_tag_map_CNB;
 250                        break;
 251                case PCI_DEVICE_ID_INTEL_IOAT_SCNB:
 252                        tag_map = ioat_tag_map_SCNB;
 253                        break;
 254                }
 255                break;
 256        case PCI_VENDOR_ID_UNISYS:
 257                switch (pdev->device) {
 258                case PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR:
 259                        tag_map = ioat_tag_map_UNISYS;
 260                        break;
 261                }
 262                break;
 263        }
 264        if (tag_map == NULL)
 265                return NULL;
 266
 267        version = readb(iobase + IOAT_VER_OFFSET);
 268        if (version == IOAT_VER_3_0)
 269                max_requesters = IOAT3_DCA_MAX_REQ;
 270        else
 271                max_requesters = IOAT_DCA_MAX_REQ;
 272
 273        dca = alloc_dca_provider(&ioat_dca_ops,
 274                        sizeof(*ioatdca) +
 275                        (sizeof(struct ioat_dca_slot) * max_requesters));
 276        if (!dca)
 277                return NULL;
 278
 279        ioatdca = dca_priv(dca);
 280        ioatdca->max_requesters = max_requesters;
 281        ioatdca->dca_base = iobase + 0x54;
 282
 283        /* copy over the APIC ID to DCA tag mapping */
 284        for (i = 0; i < IOAT_TAG_MAP_LEN; i++)
 285                ioatdca->tag_map[i] = tag_map[i];
 286
 287        err = register_dca_provider(dca, &pdev->dev);
 288        if (err) {
 289                free_dca_provider(dca);
 290                return NULL;
 291        }
 292
 293        return dca;
 294}
 295
 296
 297static int ioat2_dca_add_requester(struct dca_provider *dca, struct device *dev)
 298{
 299        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 300        struct pci_dev *pdev;
 301        int i;
 302        u16 id;
 303        u16 global_req_table;
 304
 305        /* This implementation only supports PCI-Express */
 306        if (dev->bus != &pci_bus_type)
 307                return -ENODEV;
 308        pdev = to_pci_dev(dev);
 309        id = dcaid_from_pcidev(pdev);
 310
 311        if (ioatdca->requester_count == ioatdca->max_requesters)
 312                return -ENODEV;
 313
 314        for (i = 0; i < ioatdca->max_requesters; i++) {
 315                if (ioatdca->req_slots[i].pdev == NULL) {
 316                        /* found an empty slot */
 317                        ioatdca->requester_count++;
 318                        ioatdca->req_slots[i].pdev = pdev;
 319                        ioatdca->req_slots[i].rid = id;
 320                        global_req_table =
 321                              readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
 322                        writel(id | IOAT_DCA_GREQID_VALID,
 323                               ioatdca->iobase + global_req_table + (i * 4));
 324                        return i;
 325                }
 326        }
 327        /* Error, ioatdma->requester_count is out of whack */
 328        return -EFAULT;
 329}
 330
 331static int ioat2_dca_remove_requester(struct dca_provider *dca,
 332                                      struct device *dev)
 333{
 334        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 335        struct pci_dev *pdev;
 336        int i;
 337        u16 global_req_table;
 338
 339        /* This implementation only supports PCI-Express */
 340        if (dev->bus != &pci_bus_type)
 341                return -ENODEV;
 342        pdev = to_pci_dev(dev);
 343
 344        for (i = 0; i < ioatdca->max_requesters; i++) {
 345                if (ioatdca->req_slots[i].pdev == pdev) {
 346                        global_req_table =
 347                              readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
 348                        writel(0, ioatdca->iobase + global_req_table + (i * 4));
 349                        ioatdca->req_slots[i].pdev = NULL;
 350                        ioatdca->req_slots[i].rid = 0;
 351                        ioatdca->requester_count--;
 352                        return i;
 353                }
 354        }
 355        return -ENODEV;
 356}
 357
 358static u8 ioat2_dca_get_tag(struct dca_provider *dca,
 359                            struct device *dev,
 360                            int cpu)
 361{
 362        u8 tag;
 363
 364        tag = ioat_dca_get_tag(dca, dev, cpu);
 365        tag = (~tag) & 0x1F;
 366        return tag;
 367}
 368
 369static struct dca_ops ioat2_dca_ops = {
 370        .add_requester          = ioat2_dca_add_requester,
 371        .remove_requester       = ioat2_dca_remove_requester,
 372        .get_tag                = ioat2_dca_get_tag,
 373        .dev_managed            = ioat_dca_dev_managed,
 374};
 375
 376static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
 377{
 378        int slots = 0;
 379        u32 req;
 380        u16 global_req_table;
 381
 382        global_req_table = readw(iobase + dca_offset + IOAT_DCA_GREQID_OFFSET);
 383        if (global_req_table == 0)
 384                return 0;
 385        do {
 386                req = readl(iobase + global_req_table + (slots * sizeof(u32)));
 387                slots++;
 388        } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
 389
 390        return slots;
 391}
 392
 393struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 394{
 395        struct dca_provider *dca;
 396        struct ioat_dca_priv *ioatdca;
 397        int slots;
 398        int i;
 399        int err;
 400        u32 tag_map;
 401        u16 dca_offset;
 402        u16 csi_fsb_control;
 403        u16 pcie_control;
 404        u8 bit;
 405
 406        if (!system_has_dca_enabled(pdev))
 407                return NULL;
 408
 409        dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
 410        if (dca_offset == 0)
 411                return NULL;
 412
 413        slots = ioat2_dca_count_dca_slots(iobase, dca_offset);
 414        if (slots == 0)
 415                return NULL;
 416
 417        dca = alloc_dca_provider(&ioat2_dca_ops,
 418                                 sizeof(*ioatdca)
 419                                      + (sizeof(struct ioat_dca_slot) * slots));
 420        if (!dca)
 421                return NULL;
 422
 423        ioatdca = dca_priv(dca);
 424        ioatdca->iobase = iobase;
 425        ioatdca->dca_base = iobase + dca_offset;
 426        ioatdca->max_requesters = slots;
 427
 428        /* some bios might not know to turn these on */
 429        csi_fsb_control = readw(ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
 430        if ((csi_fsb_control & IOAT_FSB_CAP_ENABLE_PREFETCH) == 0) {
 431                csi_fsb_control |= IOAT_FSB_CAP_ENABLE_PREFETCH;
 432                writew(csi_fsb_control,
 433                       ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
 434        }
 435        pcie_control = readw(ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
 436        if ((pcie_control & IOAT_PCI_CAP_ENABLE_MEMWR) == 0) {
 437                pcie_control |= IOAT_PCI_CAP_ENABLE_MEMWR;
 438                writew(pcie_control,
 439                       ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
 440        }
 441
 442
 443        /* TODO version, compatibility and configuration checks */
 444
 445        /* copy out the APIC to DCA tag map */
 446        tag_map = readl(ioatdca->dca_base + IOAT_APICID_TAG_MAP_OFFSET);
 447        for (i = 0; i < 5; i++) {
 448                bit = (tag_map >> (4 * i)) & 0x0f;
 449                if (bit < 8)
 450                        ioatdca->tag_map[i] = bit | DCA_TAG_MAP_VALID;
 451                else
 452                        ioatdca->tag_map[i] = 0;
 453        }
 454
 455        err = register_dca_provider(dca, &pdev->dev);
 456        if (err) {
 457                free_dca_provider(dca);
 458                return NULL;
 459        }
 460
 461        return dca;
 462}
 463
 464static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev)
 465{
 466        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 467        struct pci_dev *pdev;
 468        int i;
 469        u16 id;
 470        u16 global_req_table;
 471
 472        /* This implementation only supports PCI-Express */
 473        if (dev->bus != &pci_bus_type)
 474                return -ENODEV;
 475        pdev = to_pci_dev(dev);
 476        id = dcaid_from_pcidev(pdev);
 477
 478        if (ioatdca->requester_count == ioatdca->max_requesters)
 479                return -ENODEV;
 480
 481        for (i = 0; i < ioatdca->max_requesters; i++) {
 482                if (ioatdca->req_slots[i].pdev == NULL) {
 483                        /* found an empty slot */
 484                        ioatdca->requester_count++;
 485                        ioatdca->req_slots[i].pdev = pdev;
 486                        ioatdca->req_slots[i].rid = id;
 487                        global_req_table =
 488                              readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
 489                        writel(id | IOAT_DCA_GREQID_VALID,
 490                               ioatdca->iobase + global_req_table + (i * 4));
 491                        return i;
 492                }
 493        }
 494        /* Error, ioatdma->requester_count is out of whack */
 495        return -EFAULT;
 496}
 497
 498static int ioat3_dca_remove_requester(struct dca_provider *dca,
 499                                      struct device *dev)
 500{
 501        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 502        struct pci_dev *pdev;
 503        int i;
 504        u16 global_req_table;
 505
 506        /* This implementation only supports PCI-Express */
 507        if (dev->bus != &pci_bus_type)
 508                return -ENODEV;
 509        pdev = to_pci_dev(dev);
 510
 511        for (i = 0; i < ioatdca->max_requesters; i++) {
 512                if (ioatdca->req_slots[i].pdev == pdev) {
 513                        global_req_table =
 514                              readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
 515                        writel(0, ioatdca->iobase + global_req_table + (i * 4));
 516                        ioatdca->req_slots[i].pdev = NULL;
 517                        ioatdca->req_slots[i].rid = 0;
 518                        ioatdca->requester_count--;
 519                        return i;
 520                }
 521        }
 522        return -ENODEV;
 523}
 524
 525static u8 ioat3_dca_get_tag(struct dca_provider *dca,
 526                            struct device *dev,
 527                            int cpu)
 528{
 529        u8 tag;
 530
 531        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 532        int i, apic_id, bit, value;
 533        u8 entry;
 534
 535        tag = 0;
 536        apic_id = cpu_physical_id(cpu);
 537
 538        for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
 539                entry = ioatdca->tag_map[i];
 540                if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
 541                        bit = entry &
 542                                ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
 543                        value = (apic_id & (1 << bit)) ? 1 : 0;
 544                } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
 545                        bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
 546                        value = (apic_id & (1 << bit)) ? 0 : 1;
 547                } else {
 548                        value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
 549                }
 550                tag |= (value << i);
 551        }
 552
 553        return tag;
 554}
 555
 556static struct dca_ops ioat3_dca_ops = {
 557        .add_requester          = ioat3_dca_add_requester,
 558        .remove_requester       = ioat3_dca_remove_requester,
 559        .get_tag                = ioat3_dca_get_tag,
 560        .dev_managed            = ioat_dca_dev_managed,
 561};
 562
 563static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
 564{
 565        int slots = 0;
 566        u32 req;
 567        u16 global_req_table;
 568
 569        global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
 570        if (global_req_table == 0)
 571                return 0;
 572
 573        do {
 574                req = readl(iobase + global_req_table + (slots * sizeof(u32)));
 575                slots++;
 576        } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
 577
 578        return slots;
 579}
 580
 581struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 582{
 583        struct dca_provider *dca;
 584        struct ioat_dca_priv *ioatdca;
 585        int slots;
 586        int i;
 587        int err;
 588        u16 dca_offset;
 589        u16 csi_fsb_control;
 590        u16 pcie_control;
 591        u8 bit;
 592
 593        union {
 594                u64 full;
 595                struct {
 596                        u32 low;
 597                        u32 high;
 598                };
 599        } tag_map;
 600
 601        if (!system_has_dca_enabled(pdev))
 602                return NULL;
 603
 604        dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
 605        if (dca_offset == 0)
 606                return NULL;
 607
 608        slots = ioat3_dca_count_dca_slots(iobase, dca_offset);
 609        if (slots == 0)
 610                return NULL;
 611
 612        dca = alloc_dca_provider(&ioat3_dca_ops,
 613                                 sizeof(*ioatdca)
 614                                      + (sizeof(struct ioat_dca_slot) * slots));
 615        if (!dca)
 616                return NULL;
 617
 618        ioatdca = dca_priv(dca);
 619        ioatdca->iobase = iobase;
 620        ioatdca->dca_base = iobase + dca_offset;
 621        ioatdca->max_requesters = slots;
 622
 623        /* some bios might not know to turn these on */
 624        csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
 625        if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
 626                csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
 627                writew(csi_fsb_control,
 628                       ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
 629        }
 630        pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
 631        if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
 632                pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
 633                writew(pcie_control,
 634                       ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
 635        }
 636
 637
 638        /* TODO version, compatibility and configuration checks */
 639
 640        /* copy out the APIC to DCA tag map */
 641        tag_map.low =
 642                readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
 643        tag_map.high =
 644                readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
 645        for (i = 0; i < 8; i++) {
 646                bit = tag_map.full >> (8 * i);
 647                ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
 648        }
 649
 650        err = register_dca_provider(dca, &pdev->dev);
 651        if (err) {
 652                free_dca_provider(dca);
 653                return NULL;
 654        }
 655
 656        return dca;
 657}
 658