linux/drivers/ntb/ntb_hw.c
<<
>>
Prefs
   1/*
   2 * This file is provided under a dual BSD/GPLv2 license.  When using or
   3 *   redistributing this file, you may do so under either license.
   4 *
   5 *   GPL LICENSE SUMMARY
   6 *
   7 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8 *
   9 *   This program is free software; you can redistribute it and/or modify
  10 *   it under the terms of version 2 of the GNU General Public License as
  11 *   published by the Free Software Foundation.
  12 *
  13 *   BSD LICENSE
  14 *
  15 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  16 *
  17 *   Redistribution and use in source and binary forms, with or without
  18 *   modification, are permitted provided that the following conditions
  19 *   are met:
  20 *
  21 *     * Redistributions of source code must retain the above copyright
  22 *       notice, this list of conditions and the following disclaimer.
  23 *     * Redistributions in binary form must reproduce the above copy
  24 *       notice, this list of conditions and the following disclaimer in
  25 *       the documentation and/or other materials provided with the
  26 *       distribution.
  27 *     * Neither the name of Intel Corporation nor the names of its
  28 *       contributors may be used to endorse or promote products derived
  29 *       from this software without specific prior written permission.
  30 *
  31 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  32 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  33 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  34 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  35 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  37 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  41 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42 *
  43 * Intel PCIe NTB Linux driver
  44 *
  45 * Contact Information:
  46 * Jon Mason <jon.mason@intel.com>
  47 */
  48#include <linux/debugfs.h>
  49#include <linux/delay.h>
  50#include <linux/init.h>
  51#include <linux/interrupt.h>
  52#include <linux/module.h>
  53#include <linux/pci.h>
  54#include <linux/random.h>
  55#include <linux/slab.h>
  56#include "ntb_hw.h"
  57#include "ntb_regs.h"
  58
  59#define NTB_NAME        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  60#define NTB_VER         "1.0"
  61
  62MODULE_DESCRIPTION(NTB_NAME);
  63MODULE_VERSION(NTB_VER);
  64MODULE_LICENSE("Dual BSD/GPL");
  65MODULE_AUTHOR("Intel Corporation");
  66
  67static bool xeon_errata_workaround = true;
  68module_param(xeon_errata_workaround, bool, 0644);
  69MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");
  70
  71enum {
  72        NTB_CONN_TRANSPARENT = 0,
  73        NTB_CONN_B2B,
  74        NTB_CONN_RP,
  75};
  76
  77enum {
  78        NTB_DEV_USD = 0,
  79        NTB_DEV_DSD,
  80};
  81
  82enum {
  83        SNB_HW = 0,
  84        BWD_HW,
  85};
  86
  87static struct dentry *debugfs_dir;
  88
  89#define BWD_LINK_RECOVERY_TIME  500
  90
  91/* Translate memory window 0,1 to BAR 2,4 */
  92#define MW_TO_BAR(mw)   (mw * NTB_MAX_NUM_MW + 2)
  93
  94static DEFINE_PCI_DEVICE_TABLE(ntb_pci_tbl) = {
  95        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
  96        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
  97        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
  98        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
  99        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
 100        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
 101        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
 102        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
 103        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
 104        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
 105        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
 106        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
 107        {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
 108        {0}
 109};
 110MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
 111
 112/**
 113 * ntb_register_event_callback() - register event callback
 114 * @ndev: pointer to ntb_device instance
 115 * @func: callback function to register
 116 *
 117 * This function registers a callback for any HW driver events such as link
 118 * up/down, power management notices and etc.
 119 *
 120 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 121 */
 122int ntb_register_event_callback(struct ntb_device *ndev,
 123                            void (*func)(void *handle, enum ntb_hw_event event))
 124{
 125        if (ndev->event_cb)
 126                return -EINVAL;
 127
 128        ndev->event_cb = func;
 129
 130        return 0;
 131}
 132
 133/**
 134 * ntb_unregister_event_callback() - unregisters the event callback
 135 * @ndev: pointer to ntb_device instance
 136 *
 137 * This function unregisters the existing callback from transport
 138 */
 139void ntb_unregister_event_callback(struct ntb_device *ndev)
 140{
 141        ndev->event_cb = NULL;
 142}
 143
 144/**
 145 * ntb_register_db_callback() - register a callback for doorbell interrupt
 146 * @ndev: pointer to ntb_device instance
 147 * @idx: doorbell index to register callback, zero based
 148 * @data: pointer to be returned to caller with every callback
 149 * @func: callback function to register
 150 *
 151 * This function registers a callback function for the doorbell interrupt
 152 * on the primary side. The function will unmask the doorbell as well to
 153 * allow interrupt.
 154 *
 155 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 156 */
 157int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
 158                             void *data, void (*func)(void *data, int db_num))
 159{
 160        unsigned long mask;
 161
 162        if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
 163                dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
 164                return -EINVAL;
 165        }
 166
 167        ndev->db_cb[idx].callback = func;
 168        ndev->db_cb[idx].data = data;
 169
 170        /* unmask interrupt */
 171        mask = readw(ndev->reg_ofs.ldb_mask);
 172        clear_bit(idx * ndev->bits_per_vector, &mask);
 173        writew(mask, ndev->reg_ofs.ldb_mask);
 174
 175        return 0;
 176}
 177
 178/**
 179 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
 180 * @ndev: pointer to ntb_device instance
 181 * @idx: doorbell index to register callback, zero based
 182 *
 183 * This function unregisters a callback function for the doorbell interrupt
 184 * on the primary side. The function will also mask the said doorbell.
 185 */
 186void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
 187{
 188        unsigned long mask;
 189
 190        if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
 191                return;
 192
 193        mask = readw(ndev->reg_ofs.ldb_mask);
 194        set_bit(idx * ndev->bits_per_vector, &mask);
 195        writew(mask, ndev->reg_ofs.ldb_mask);
 196
 197        ndev->db_cb[idx].callback = NULL;
 198}
 199
 200/**
 201 * ntb_find_transport() - find the transport pointer
 202 * @transport: pointer to pci device
 203 *
 204 * Given the pci device pointer, return the transport pointer passed in when
 205 * the transport attached when it was inited.
 206 *
 207 * RETURNS: pointer to transport.
 208 */
 209void *ntb_find_transport(struct pci_dev *pdev)
 210{
 211        struct ntb_device *ndev = pci_get_drvdata(pdev);
 212        return ndev->ntb_transport;
 213}
 214
 215/**
 216 * ntb_register_transport() - Register NTB transport with NTB HW driver
 217 * @transport: transport identifier
 218 *
 219 * This function allows a transport to reserve the hardware driver for
 220 * NTB usage.
 221 *
 222 * RETURNS: pointer to ntb_device, NULL on error.
 223 */
 224struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
 225{
 226        struct ntb_device *ndev = pci_get_drvdata(pdev);
 227
 228        if (ndev->ntb_transport)
 229                return NULL;
 230
 231        ndev->ntb_transport = transport;
 232        return ndev;
 233}
 234
 235/**
 236 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
 237 * @ndev - ntb_device of the transport to be freed
 238 *
 239 * This function unregisters the transport from the HW driver and performs any
 240 * necessary cleanups.
 241 */
 242void ntb_unregister_transport(struct ntb_device *ndev)
 243{
 244        int i;
 245
 246        if (!ndev->ntb_transport)
 247                return;
 248
 249        for (i = 0; i < ndev->max_cbs; i++)
 250                ntb_unregister_db_callback(ndev, i);
 251
 252        ntb_unregister_event_callback(ndev);
 253        ndev->ntb_transport = NULL;
 254}
 255
 256/**
 257 * ntb_write_local_spad() - write to the secondary scratchpad register
 258 * @ndev: pointer to ntb_device instance
 259 * @idx: index to the scratchpad register, 0 based
 260 * @val: the data value to put into the register
 261 *
 262 * This function allows writing of a 32bit value to the indexed scratchpad
 263 * register. This writes over the data mirrored to the local scratchpad register
 264 * by the remote system.
 265 *
 266 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 267 */
 268int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
 269{
 270        if (idx >= ndev->limits.max_spads)
 271                return -EINVAL;
 272
 273        dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
 274                val, idx);
 275        writel(val, ndev->reg_ofs.spad_read + idx * 4);
 276
 277        return 0;
 278}
 279
 280/**
 281 * ntb_read_local_spad() - read from the primary scratchpad register
 282 * @ndev: pointer to ntb_device instance
 283 * @idx: index to scratchpad register, 0 based
 284 * @val: pointer to 32bit integer for storing the register value
 285 *
 286 * This function allows reading of the 32bit scratchpad register on
 287 * the primary (internal) side.  This allows the local system to read data
 288 * written and mirrored to the scratchpad register by the remote system.
 289 *
 290 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 291 */
 292int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 293{
 294        if (idx >= ndev->limits.max_spads)
 295                return -EINVAL;
 296
 297        *val = readl(ndev->reg_ofs.spad_write + idx * 4);
 298        dev_dbg(&ndev->pdev->dev,
 299                "Reading %x from local scratch pad index %d\n", *val, idx);
 300
 301        return 0;
 302}
 303
 304/**
 305 * ntb_write_remote_spad() - write to the secondary scratchpad register
 306 * @ndev: pointer to ntb_device instance
 307 * @idx: index to the scratchpad register, 0 based
 308 * @val: the data value to put into the register
 309 *
 310 * This function allows writing of a 32bit value to the indexed scratchpad
 311 * register. The register resides on the secondary (external) side.  This allows
 312 * the local system to write data to be mirrored to the remote systems
 313 * scratchpad register.
 314 *
 315 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 316 */
 317int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
 318{
 319        if (idx >= ndev->limits.max_spads)
 320                return -EINVAL;
 321
 322        dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
 323                val, idx);
 324        writel(val, ndev->reg_ofs.spad_write + idx * 4);
 325
 326        return 0;
 327}
 328
 329/**
 330 * ntb_read_remote_spad() - read from the primary scratchpad register
 331 * @ndev: pointer to ntb_device instance
 332 * @idx: index to scratchpad register, 0 based
 333 * @val: pointer to 32bit integer for storing the register value
 334 *
 335 * This function allows reading of the 32bit scratchpad register on
 336 * the primary (internal) side.  This alloows the local system to read the data
 337 * it wrote to be mirrored on the remote system.
 338 *
 339 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 340 */
 341int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 342{
 343        if (idx >= ndev->limits.max_spads)
 344                return -EINVAL;
 345
 346        *val = readl(ndev->reg_ofs.spad_read + idx * 4);
 347        dev_dbg(&ndev->pdev->dev,
 348                "Reading %x from remote scratch pad index %d\n", *val, idx);
 349
 350        return 0;
 351}
 352
 353/**
 354 * ntb_get_mw_base() - get addr for the NTB memory window
 355 * @ndev: pointer to ntb_device instance
 356 * @mw: memory window number
 357 *
 358 * This function provides the base address of the memory window specified.
 359 *
 360 * RETURNS: address, or NULL on error.
 361 */
 362resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
 363{
 364        if (mw >= ntb_max_mw(ndev))
 365                return 0;
 366
 367        return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
 368}
 369
 370/**
 371 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
 372 * @ndev: pointer to ntb_device instance
 373 * @mw: memory window number
 374 *
 375 * This function provides the base virtual address of the memory window
 376 * specified.
 377 *
 378 * RETURNS: pointer to virtual address, or NULL on error.
 379 */
 380void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
 381{
 382        if (mw >= ntb_max_mw(ndev))
 383                return NULL;
 384
 385        return ndev->mw[mw].vbase;
 386}
 387
 388/**
 389 * ntb_get_mw_size() - return size of NTB memory window
 390 * @ndev: pointer to ntb_device instance
 391 * @mw: memory window number
 392 *
 393 * This function provides the physical size of the memory window specified
 394 *
 395 * RETURNS: the size of the memory window or zero on error
 396 */
 397u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
 398{
 399        if (mw >= ntb_max_mw(ndev))
 400                return 0;
 401
 402        return ndev->mw[mw].bar_sz;
 403}
 404
 405/**
 406 * ntb_set_mw_addr - set the memory window address
 407 * @ndev: pointer to ntb_device instance
 408 * @mw: memory window number
 409 * @addr: base address for data
 410 *
 411 * This function sets the base physical address of the memory window.  This
 412 * memory address is where data from the remote system will be transfered into
 413 * or out of depending on how the transport is configured.
 414 */
 415void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
 416{
 417        if (mw >= ntb_max_mw(ndev))
 418                return;
 419
 420        dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
 421                MW_TO_BAR(mw));
 422
 423        ndev->mw[mw].phys_addr = addr;
 424
 425        switch (MW_TO_BAR(mw)) {
 426        case NTB_BAR_23:
 427                writeq(addr, ndev->reg_ofs.bar2_xlat);
 428                break;
 429        case NTB_BAR_45:
 430                writeq(addr, ndev->reg_ofs.bar4_xlat);
 431                break;
 432        }
 433}
 434
 435/**
 436 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
 437 * @ndev: pointer to ntb_device instance
 438 * @db: doorbell to ring
 439 *
 440 * This function allows triggering of a doorbell on the secondary/external
 441 * side that will initiate an interrupt on the remote host
 442 *
 443 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 444 */
 445void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
 446{
 447        dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
 448
 449        if (ndev->hw_type == BWD_HW)
 450                writeq((u64) 1 << db, ndev->reg_ofs.rdb);
 451        else
 452                writew(((1 << ndev->bits_per_vector) - 1) <<
 453                       (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
 454}
 455
 456static void bwd_recover_link(struct ntb_device *ndev)
 457{
 458        u32 status;
 459
 460        /* Driver resets the NTB ModPhy lanes - magic! */
 461        writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
 462        writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
 463        writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
 464        writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
 465
 466        /* Driver waits 100ms to allow the NTB ModPhy to settle */
 467        msleep(100);
 468
 469        /* Clear AER Errors, write to clear */
 470        status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
 471        dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
 472        status &= PCI_ERR_COR_REP_ROLL;
 473        writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
 474
 475        /* Clear unexpected electrical idle event in LTSSM, write to clear */
 476        status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
 477        dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
 478        status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
 479        writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
 480
 481        /* Clear DeSkew Buffer error, write to clear */
 482        status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
 483        dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
 484        status |= BWD_DESKEWSTS_DBERR;
 485        writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
 486
 487        status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 488        dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
 489        status &= BWD_IBIST_ERR_OFLOW;
 490        writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 491
 492        /* Releases the NTB state machine to allow the link to retrain */
 493        status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 494        dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
 495        status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
 496        writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 497}
 498
 499static void ntb_link_event(struct ntb_device *ndev, int link_state)
 500{
 501        unsigned int event;
 502
 503        if (ndev->link_status == link_state)
 504                return;
 505
 506        if (link_state == NTB_LINK_UP) {
 507                u16 status;
 508
 509                dev_info(&ndev->pdev->dev, "Link Up\n");
 510                ndev->link_status = NTB_LINK_UP;
 511                event = NTB_EVENT_HW_LINK_UP;
 512
 513                if (ndev->hw_type == BWD_HW ||
 514                    ndev->conn_type == NTB_CONN_TRANSPARENT)
 515                        status = readw(ndev->reg_ofs.lnk_stat);
 516                else {
 517                        int rc = pci_read_config_word(ndev->pdev,
 518                                                      SNB_LINK_STATUS_OFFSET,
 519                                                      &status);
 520                        if (rc)
 521                                return;
 522                }
 523
 524                ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
 525                ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
 526                dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
 527                         ndev->link_width, ndev->link_speed);
 528        } else {
 529                dev_info(&ndev->pdev->dev, "Link Down\n");
 530                ndev->link_status = NTB_LINK_DOWN;
 531                event = NTB_EVENT_HW_LINK_DOWN;
 532                /* Don't modify link width/speed, we need it in link recovery */
 533        }
 534
 535        /* notify the upper layer if we have an event change */
 536        if (ndev->event_cb)
 537                ndev->event_cb(ndev->ntb_transport, event);
 538}
 539
 540static int ntb_link_status(struct ntb_device *ndev)
 541{
 542        int link_state;
 543
 544        if (ndev->hw_type == BWD_HW) {
 545                u32 ntb_cntl;
 546
 547                ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
 548                if (ntb_cntl & BWD_CNTL_LINK_DOWN)
 549                        link_state = NTB_LINK_DOWN;
 550                else
 551                        link_state = NTB_LINK_UP;
 552        } else {
 553                u16 status;
 554                int rc;
 555
 556                rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
 557                                          &status);
 558                if (rc)
 559                        return rc;
 560
 561                if (status & NTB_LINK_STATUS_ACTIVE)
 562                        link_state = NTB_LINK_UP;
 563                else
 564                        link_state = NTB_LINK_DOWN;
 565        }
 566
 567        ntb_link_event(ndev, link_state);
 568
 569        return 0;
 570}
 571
 572static void bwd_link_recovery(struct work_struct *work)
 573{
 574        struct ntb_device *ndev = container_of(work, struct ntb_device,
 575                                               lr_timer.work);
 576        u32 status32;
 577
 578        bwd_recover_link(ndev);
 579        /* There is a potential race between the 2 NTB devices recovering at the
 580         * same time.  If the times are the same, the link will not recover and
 581         * the driver will be stuck in this loop forever.  Add a random interval
 582         * to the recovery time to prevent this race.
 583         */
 584        msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
 585
 586        status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 587        if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
 588                goto retry;
 589
 590        status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 591        if (status32 & BWD_IBIST_ERR_OFLOW)
 592                goto retry;
 593
 594        status32 = readl(ndev->reg_ofs.lnk_cntl);
 595        if (!(status32 & BWD_CNTL_LINK_DOWN)) {
 596                unsigned char speed, width;
 597                u16 status16;
 598
 599                status16 = readw(ndev->reg_ofs.lnk_stat);
 600                width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
 601                speed = (status16 & NTB_LINK_SPEED_MASK);
 602                if (ndev->link_width != width || ndev->link_speed != speed)
 603                        goto retry;
 604        }
 605
 606        schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 607        return;
 608
 609retry:
 610        schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
 611}
 612
 613/* BWD doesn't have link status interrupt, poll on that platform */
 614static void bwd_link_poll(struct work_struct *work)
 615{
 616        struct ntb_device *ndev = container_of(work, struct ntb_device,
 617                                               hb_timer.work);
 618        unsigned long ts = jiffies;
 619
 620        /* If we haven't gotten an interrupt in a while, check the BWD link
 621         * status bit
 622         */
 623        if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
 624                int rc = ntb_link_status(ndev);
 625                if (rc)
 626                        dev_err(&ndev->pdev->dev,
 627                                "Error determining link status\n");
 628
 629                /* Check to see if a link error is the cause of the link down */
 630                if (ndev->link_status == NTB_LINK_DOWN) {
 631                        u32 status32 = readl(ndev->reg_base +
 632                                             BWD_LTSSMSTATEJMP_OFFSET);
 633                        if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
 634                                schedule_delayed_work(&ndev->lr_timer, 0);
 635                                return;
 636                        }
 637                }
 638        }
 639
 640        schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 641}
 642
 643static int ntb_xeon_setup(struct ntb_device *ndev)
 644{
 645        int rc;
 646        u8 val;
 647
 648        ndev->hw_type = SNB_HW;
 649
 650        rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val);
 651        if (rc)
 652                return rc;
 653
 654        if (val & SNB_PPD_DEV_TYPE)
 655                ndev->dev_type = NTB_DEV_USD;
 656        else
 657                ndev->dev_type = NTB_DEV_DSD;
 658
 659        switch (val & SNB_PPD_CONN_TYPE) {
 660        case NTB_CONN_B2B:
 661                dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
 662                ndev->conn_type = NTB_CONN_B2B;
 663                ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 664                ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
 665                ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
 666                ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
 667                ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
 668                ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
 669
 670                /* There is a Xeon hardware errata related to writes to
 671                 * SDOORBELL or B2BDOORBELL in conjunction with inbound access
 672                 * to NTB MMIO Space, which may hang the system.  To workaround
 673                 * this use the second memory window to access the interrupt and
 674                 * scratch pad registers on the remote system.
 675                 */
 676                if (xeon_errata_workaround) {
 677                        if (!ndev->mw[1].bar_sz)
 678                                return -EINVAL;
 679
 680                        ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
 681                        ndev->reg_ofs.spad_write = ndev->mw[1].vbase +
 682                                                   SNB_SPAD_OFFSET;
 683                        ndev->reg_ofs.rdb = ndev->mw[1].vbase +
 684                                            SNB_PDOORBELL_OFFSET;
 685
 686                        /* Set the Limit register to 4k, the minimum size, to
 687                         * prevent an illegal access
 688                         */
 689                        writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
 690                               SNB_PBAR4LMT_OFFSET);
 691                } else {
 692                        ndev->limits.max_mw = SNB_MAX_MW;
 693                        ndev->reg_ofs.spad_write = ndev->reg_base +
 694                                                   SNB_B2B_SPAD_OFFSET;
 695                        ndev->reg_ofs.rdb = ndev->reg_base +
 696                                            SNB_B2B_DOORBELL_OFFSET;
 697
 698                        /* Disable the Limit register, just incase it is set to
 699                         * something silly
 700                         */
 701                        writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
 702                }
 703
 704                /* The Xeon errata workaround requires setting SBAR Base
 705                 * addresses to known values, so that the PBAR XLAT can be
 706                 * pointed at SBAR0 of the remote system.
 707                 */
 708                if (ndev->dev_type == NTB_DEV_USD) {
 709                        writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
 710                               SNB_PBAR2XLAT_OFFSET);
 711                        if (xeon_errata_workaround)
 712                                writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
 713                                       SNB_PBAR4XLAT_OFFSET);
 714                        else {
 715                                writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
 716                                       SNB_PBAR4XLAT_OFFSET);
 717                                /* B2B_XLAT_OFFSET is a 64bit register, but can
 718                                 * only take 32bit writes
 719                                 */
 720                                writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
 721                                       ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
 722                                writel(SNB_MBAR01_DSD_ADDR >> 32,
 723                                       ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
 724                        }
 725
 726                        writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
 727                               SNB_SBAR0BASE_OFFSET);
 728                        writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
 729                               SNB_SBAR2BASE_OFFSET);
 730                        writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
 731                               SNB_SBAR4BASE_OFFSET);
 732                } else {
 733                        writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
 734                               SNB_PBAR2XLAT_OFFSET);
 735                        if (xeon_errata_workaround)
 736                                writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
 737                                       SNB_PBAR4XLAT_OFFSET);
 738                        else {
 739                                writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
 740                                       SNB_PBAR4XLAT_OFFSET);
 741                                /* B2B_XLAT_OFFSET is a 64bit register, but can
 742                                 * only take 32bit writes
 743                                 */
 744                                writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
 745                                       ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
 746                                writel(SNB_MBAR01_USD_ADDR >> 32,
 747                                       ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
 748                        }
 749                        writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
 750                               SNB_SBAR0BASE_OFFSET);
 751                        writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
 752                               SNB_SBAR2BASE_OFFSET);
 753                        writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
 754                               SNB_SBAR4BASE_OFFSET);
 755                }
 756                break;
 757        case NTB_CONN_RP:
 758                dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
 759                ndev->conn_type = NTB_CONN_RP;
 760
 761                if (xeon_errata_workaround) {
 762                        dev_err(&ndev->pdev->dev, 
 763                                "NTB-RP disabled due to hardware errata.  To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");
 764                        return -EINVAL;
 765                }
 766
 767                /* Scratch pads need to have exclusive access from the primary
 768                 * or secondary side.  Halve the num spads so that each side can
 769                 * have an equal amount.
 770                 */
 771                ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
 772                /* Note: The SDOORBELL is the cause of the errata.  You REALLY
 773                 * don't want to touch it.
 774                 */
 775                ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
 776                ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 777                ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
 778                /* Offset the start of the spads to correspond to whether it is
 779                 * primary or secondary
 780                 */
 781                ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
 782                                           ndev->limits.max_spads * 4;
 783                ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
 784                ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
 785                ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
 786                ndev->limits.max_mw = SNB_MAX_MW;
 787                break;
 788        case NTB_CONN_TRANSPARENT:
 789                dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
 790                ndev->conn_type = NTB_CONN_TRANSPARENT;
 791                /* Scratch pads need to have exclusive access from the primary
 792                 * or secondary side.  Halve the num spads so that each side can
 793                 * have an equal amount.
 794                 */
 795                ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
 796                ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 797                ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
 798                ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
 799                ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
 800                /* Offset the start of the spads to correspond to whether it is
 801                 * primary or secondary
 802                 */
 803                ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
 804                                          ndev->limits.max_spads * 4;
 805                ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
 806                ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
 807
 808                ndev->limits.max_mw = SNB_MAX_MW;
 809                break;
 810        default:
 811                /* Most likely caused by the remote NTB-RP device not being
 812                 * configured
 813                 */
 814                dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);
 815                return -EINVAL;
 816        }
 817
 818        ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
 819        ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
 820        ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
 821
 822        ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 823        ndev->limits.msix_cnt = SNB_MSIX_CNT;
 824        ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
 825
 826        return 0;
 827}
 828
 829static int ntb_bwd_setup(struct ntb_device *ndev)
 830{
 831        int rc;
 832        u32 val;
 833
 834        ndev->hw_type = BWD_HW;
 835
 836        rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
 837        if (rc)
 838                return rc;
 839
 840        switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
 841        case NTB_CONN_B2B:
 842                ndev->conn_type = NTB_CONN_B2B;
 843                break;
 844        case NTB_CONN_RP:
 845        default:
 846                dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
 847                return -EINVAL;
 848        }
 849
 850        if (val & BWD_PPD_DEV_TYPE)
 851                ndev->dev_type = NTB_DEV_DSD;
 852        else
 853                ndev->dev_type = NTB_DEV_USD;
 854
 855        /* Initiate PCI-E link training */
 856        rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
 857                                    val | BWD_PPD_INIT_LINK);
 858        if (rc)
 859                return rc;
 860
 861        ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
 862        ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
 863        ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
 864        ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
 865        ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
 866        ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
 867        ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
 868        ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
 869        ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
 870        ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
 871        ndev->limits.max_mw = BWD_MAX_MW;
 872        ndev->limits.max_spads = BWD_MAX_SPADS;
 873        ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
 874        ndev->limits.msix_cnt = BWD_MSIX_CNT;
 875        ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
 876
 877        /* Since bwd doesn't have a link interrupt, setup a poll timer */
 878        INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
 879        INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
 880        schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 881
 882        return 0;
 883}
 884
 885static int ntb_device_setup(struct ntb_device *ndev)
 886{
 887        int rc;
 888
 889        switch (ndev->pdev->device) {
 890        case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 891        case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 892        case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 893        case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 894        case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 895        case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 896        case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 897        case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 898        case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 899        case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 900        case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 901        case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 902                rc = ntb_xeon_setup(ndev);
 903                break;
 904        case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
 905                rc = ntb_bwd_setup(ndev);
 906                break;
 907        default:
 908                rc = -ENODEV;
 909        }
 910
 911        if (rc)
 912                return rc;
 913
 914        dev_info(&ndev->pdev->dev, "Device Type = %s\n",
 915                 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
 916
 917        if (ndev->conn_type == NTB_CONN_B2B)
 918                /* Enable Bus Master and Memory Space on the secondary side */
 919                writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
 920                       ndev->reg_ofs.spci_cmd);
 921
 922        return 0;
 923}
 924
 925static void ntb_device_free(struct ntb_device *ndev)
 926{
 927        if (ndev->hw_type == BWD_HW) {
 928                cancel_delayed_work_sync(&ndev->hb_timer);
 929                cancel_delayed_work_sync(&ndev->lr_timer);
 930        }
 931}
 932
 933static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
 934{
 935        struct ntb_db_cb *db_cb = data;
 936        struct ntb_device *ndev = db_cb->ndev;
 937
 938        dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
 939                db_cb->db_num);
 940
 941        if (db_cb->callback)
 942                db_cb->callback(db_cb->data, db_cb->db_num);
 943
 944        /* No need to check for the specific HB irq, any interrupt means
 945         * we're connected.
 946         */
 947        ndev->last_ts = jiffies;
 948
 949        writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
 950
 951        return IRQ_HANDLED;
 952}
 953
 954static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
 955{
 956        struct ntb_db_cb *db_cb = data;
 957        struct ntb_device *ndev = db_cb->ndev;
 958
 959        dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
 960                db_cb->db_num);
 961
 962        if (db_cb->callback)
 963                db_cb->callback(db_cb->data, db_cb->db_num);
 964
 965        /* On Sandybridge, there are 16 bits in the interrupt register
 966         * but only 4 vectors.  So, 5 bits are assigned to the first 3
 967         * vectors, with the 4th having a single bit for link
 968         * interrupts.
 969         */
 970        writew(((1 << ndev->bits_per_vector) - 1) <<
 971               (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
 972
 973        return IRQ_HANDLED;
 974}
 975
 976/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
 977static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
 978{
 979        struct ntb_device *ndev = dev;
 980        int rc;
 981
 982        dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
 983
 984        rc = ntb_link_status(ndev);
 985        if (rc)
 986                dev_err(&ndev->pdev->dev, "Error determining link status\n");
 987
 988        /* bit 15 is always the link bit */
 989        writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb);
 990
 991        return IRQ_HANDLED;
 992}
 993
 994static irqreturn_t ntb_interrupt(int irq, void *dev)
 995{
 996        struct ntb_device *ndev = dev;
 997        unsigned int i = 0;
 998
 999        if (ndev->hw_type == BWD_HW) {
1000                u64 ldb = readq(ndev->reg_ofs.ldb);
1001
1002                dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
1003
1004                while (ldb) {
1005                        i = __ffs(ldb);
1006                        ldb &= ldb - 1;
1007                        bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1008                }
1009        } else {
1010                u16 ldb = readw(ndev->reg_ofs.ldb);
1011
1012                dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
1013
1014                if (ldb & SNB_DB_HW_LINK) {
1015                        xeon_event_msix_irq(irq, dev);
1016                        ldb &= ~SNB_DB_HW_LINK;
1017                }
1018
1019                while (ldb) {
1020                        i = __ffs(ldb);
1021                        ldb &= ldb - 1;
1022                        xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1023                }
1024        }
1025
1026        return IRQ_HANDLED;
1027}
1028
1029static int ntb_setup_msix(struct ntb_device *ndev)
1030{
1031        struct pci_dev *pdev = ndev->pdev;
1032        struct msix_entry *msix;
1033        int msix_entries;
1034        int rc, i;
1035        u16 val;
1036
1037        if (!pdev->msix_cap) {
1038                rc = -EIO;
1039                goto err;
1040        }
1041
1042        rc = pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &val);
1043        if (rc)
1044                goto err;
1045
1046        msix_entries = msix_table_size(val);
1047        if (msix_entries > ndev->limits.msix_cnt) {
1048                rc = -EINVAL;
1049                goto err;
1050        }
1051
1052        ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
1053                                     GFP_KERNEL);
1054        if (!ndev->msix_entries) {
1055                rc = -ENOMEM;
1056                goto err;
1057        }
1058
1059        for (i = 0; i < msix_entries; i++)
1060                ndev->msix_entries[i].entry = i;
1061
1062        rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
1063        if (rc < 0)
1064                goto err1;
1065        if (rc > 0) {
1066                /* On SNB, the link interrupt is always tied to 4th vector.  If
1067                 * we can't get all 4, then we can't use MSI-X.
1068                 */
1069                if (ndev->hw_type != BWD_HW) {
1070                        rc = -EIO;
1071                        goto err1;
1072                }
1073
1074                dev_warn(&pdev->dev,
1075                         "Only %d MSI-X vectors.  Limiting the number of queues to that number.\n",
1076                         rc);
1077                msix_entries = rc;
1078        }
1079
1080        for (i = 0; i < msix_entries; i++) {
1081                msix = &ndev->msix_entries[i];
1082                WARN_ON(!msix->vector);
1083
1084                /* Use the last MSI-X vector for Link status */
1085                if (ndev->hw_type == BWD_HW) {
1086                        rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
1087                                         "ntb-callback-msix", &ndev->db_cb[i]);
1088                        if (rc)
1089                                goto err2;
1090                } else {
1091                        if (i == msix_entries - 1) {
1092                                rc = request_irq(msix->vector,
1093                                                 xeon_event_msix_irq, 0,
1094                                                 "ntb-event-msix", ndev);
1095                                if (rc)
1096                                        goto err2;
1097                        } else {
1098                                rc = request_irq(msix->vector,
1099                                                 xeon_callback_msix_irq, 0,
1100                                                 "ntb-callback-msix",
1101                                                 &ndev->db_cb[i]);
1102                                if (rc)
1103                                        goto err2;
1104                        }
1105                }
1106        }
1107
1108        ndev->num_msix = msix_entries;
1109        if (ndev->hw_type == BWD_HW)
1110                ndev->max_cbs = msix_entries;
1111        else
1112                ndev->max_cbs = msix_entries - 1;
1113
1114        return 0;
1115
1116err2:
1117        while (--i >= 0) {
1118                msix = &ndev->msix_entries[i];
1119                if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1120                        free_irq(msix->vector, ndev);
1121                else
1122                        free_irq(msix->vector, &ndev->db_cb[i]);
1123        }
1124        pci_disable_msix(pdev);
1125err1:
1126        kfree(ndev->msix_entries);
1127        dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
1128err:
1129        ndev->num_msix = 0;
1130        return rc;
1131}
1132
1133static int ntb_setup_msi(struct ntb_device *ndev)
1134{
1135        struct pci_dev *pdev = ndev->pdev;
1136        int rc;
1137
1138        rc = pci_enable_msi(pdev);
1139        if (rc)
1140                return rc;
1141
1142        rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
1143        if (rc) {
1144                pci_disable_msi(pdev);
1145                dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
1146                return rc;
1147        }
1148
1149        return 0;
1150}
1151
1152static int ntb_setup_intx(struct ntb_device *ndev)
1153{
1154        struct pci_dev *pdev = ndev->pdev;
1155        int rc;
1156
1157        pci_msi_off(pdev);
1158
1159        /* Verify intx is enabled */
1160        pci_intx(pdev, 1);
1161
1162        rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
1163                         ndev);
1164        if (rc)
1165                return rc;
1166
1167        return 0;
1168}
1169
1170static int ntb_setup_interrupts(struct ntb_device *ndev)
1171{
1172        int rc;
1173
1174        /* On BWD, disable all interrupts.  On SNB, disable all but Link
1175         * Interrupt.  The rest will be unmasked as callbacks are registered.
1176         */
1177        if (ndev->hw_type == BWD_HW)
1178                writeq(~0, ndev->reg_ofs.ldb_mask);
1179        else
1180                writew(~(1 << ndev->limits.max_db_bits),
1181                       ndev->reg_ofs.ldb_mask);
1182
1183        rc = ntb_setup_msix(ndev);
1184        if (!rc)
1185                goto done;
1186
1187        ndev->bits_per_vector = 1;
1188        ndev->max_cbs = ndev->limits.max_db_bits;
1189
1190        rc = ntb_setup_msi(ndev);
1191        if (!rc)
1192                goto done;
1193
1194        rc = ntb_setup_intx(ndev);
1195        if (rc) {
1196                dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1197                return rc;
1198        }
1199
1200done:
1201        return 0;
1202}
1203
1204static void ntb_free_interrupts(struct ntb_device *ndev)
1205{
1206        struct pci_dev *pdev = ndev->pdev;
1207
1208        /* mask interrupts */
1209        if (ndev->hw_type == BWD_HW)
1210                writeq(~0, ndev->reg_ofs.ldb_mask);
1211        else
1212                writew(~0, ndev->reg_ofs.ldb_mask);
1213
1214        if (ndev->num_msix) {
1215                struct msix_entry *msix;
1216                u32 i;
1217
1218                for (i = 0; i < ndev->num_msix; i++) {
1219                        msix = &ndev->msix_entries[i];
1220                        if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1221                                free_irq(msix->vector, ndev);
1222                        else
1223                                free_irq(msix->vector, &ndev->db_cb[i]);
1224                }
1225                pci_disable_msix(pdev);
1226        } else {
1227                free_irq(pdev->irq, ndev);
1228
1229                if (pci_dev_msi_enabled(pdev))
1230                        pci_disable_msi(pdev);
1231        }
1232}
1233
1234static int ntb_create_callbacks(struct ntb_device *ndev)
1235{
1236        int i;
1237
1238        /* Chicken-egg issue.  We won't know how many callbacks are necessary
1239         * until we see how many MSI-X vectors we get, but these pointers need
1240         * to be passed into the MSI-X register function.  So, we allocate the
1241         * max, knowing that they might not all be used, to work around this.
1242         */
1243        ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
1244                              sizeof(struct ntb_db_cb),
1245                              GFP_KERNEL);
1246        if (!ndev->db_cb)
1247                return -ENOMEM;
1248
1249        for (i = 0; i < ndev->limits.max_db_bits; i++) {
1250                ndev->db_cb[i].db_num = i;
1251                ndev->db_cb[i].ndev = ndev;
1252        }
1253
1254        return 0;
1255}
1256
1257static void ntb_free_callbacks(struct ntb_device *ndev)
1258{
1259        int i;
1260
1261        for (i = 0; i < ndev->limits.max_db_bits; i++)
1262                ntb_unregister_db_callback(ndev, i);
1263
1264        kfree(ndev->db_cb);
1265}
1266
1267static void ntb_setup_debugfs(struct ntb_device *ndev)
1268{
1269        if (!debugfs_initialized())
1270                return;
1271
1272        if (!debugfs_dir)
1273                debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1274
1275        ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
1276                                               debugfs_dir);
1277}
1278
1279static void ntb_free_debugfs(struct ntb_device *ndev)
1280{
1281        debugfs_remove_recursive(ndev->debugfs_dir);
1282
1283        if (debugfs_dir && simple_empty(debugfs_dir)) {
1284                debugfs_remove_recursive(debugfs_dir);
1285                debugfs_dir = NULL;
1286        }
1287}
1288
1289static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1290{
1291        struct ntb_device *ndev;
1292        int rc, i;
1293
1294        ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
1295        if (!ndev)
1296                return -ENOMEM;
1297
1298        ndev->pdev = pdev;
1299        ndev->link_status = NTB_LINK_DOWN;
1300        pci_set_drvdata(pdev, ndev);
1301        ntb_setup_debugfs(ndev);
1302
1303        rc = pci_enable_device(pdev);
1304        if (rc)
1305                goto err;
1306
1307        pci_set_master(ndev->pdev);
1308
1309        rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
1310        if (rc)
1311                goto err1;
1312
1313        ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
1314        if (!ndev->reg_base) {
1315                dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
1316                rc = -EIO;
1317                goto err2;
1318        }
1319
1320        for (i = 0; i < NTB_MAX_NUM_MW; i++) {
1321                ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1322                ndev->mw[i].vbase =
1323                    ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
1324                               ndev->mw[i].bar_sz);
1325                dev_info(&pdev->dev, "MW %d size %llu\n", i,
1326                         (unsigned long long) ndev->mw[i].bar_sz);
1327                if (!ndev->mw[i].vbase) {
1328                        dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
1329                                 MW_TO_BAR(i));
1330                        rc = -EIO;
1331                        goto err3;
1332                }
1333        }
1334
1335        rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1336        if (rc) {
1337                rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1338                if (rc)
1339                        goto err3;
1340
1341                dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1342        }
1343
1344        rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1345        if (rc) {
1346                rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1347                if (rc)
1348                        goto err3;
1349
1350                dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1351        }
1352
1353        rc = ntb_device_setup(ndev);
1354        if (rc)
1355                goto err3;
1356
1357        rc = ntb_create_callbacks(ndev);
1358        if (rc)
1359                goto err4;
1360
1361        rc = ntb_setup_interrupts(ndev);
1362        if (rc)
1363                goto err5;
1364
1365        /* The scratchpad registers keep the values between rmmod/insmod,
1366         * blast them now
1367         */
1368        for (i = 0; i < ndev->limits.max_spads; i++) {
1369                ntb_write_local_spad(ndev, i, 0);
1370                ntb_write_remote_spad(ndev, i, 0);
1371        }
1372
1373        rc = ntb_transport_init(pdev);
1374        if (rc)
1375                goto err6;
1376
1377        /* Let's bring the NTB link up */
1378        writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,
1379               ndev->reg_ofs.lnk_cntl);
1380
1381        return 0;
1382
1383err6:
1384        ntb_free_interrupts(ndev);
1385err5:
1386        ntb_free_callbacks(ndev);
1387err4:
1388        ntb_device_free(ndev);
1389err3:
1390        for (i--; i >= 0; i--)
1391                iounmap(ndev->mw[i].vbase);
1392        iounmap(ndev->reg_base);
1393err2:
1394        pci_release_selected_regions(pdev, NTB_BAR_MASK);
1395err1:
1396        pci_disable_device(pdev);
1397err:
1398        ntb_free_debugfs(ndev);
1399        kfree(ndev);
1400
1401        dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1402        return rc;
1403}
1404
1405static void ntb_pci_remove(struct pci_dev *pdev)
1406{
1407        struct ntb_device *ndev = pci_get_drvdata(pdev);
1408        int i;
1409        u32 ntb_cntl;
1410
1411        /* Bring NTB link down */
1412        ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1413        ntb_cntl |= NTB_CNTL_LINK_DISABLE;
1414        writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1415
1416        ntb_transport_free(ndev->ntb_transport);
1417
1418        ntb_free_interrupts(ndev);
1419        ntb_free_callbacks(ndev);
1420        ntb_device_free(ndev);
1421
1422        for (i = 0; i < NTB_MAX_NUM_MW; i++)
1423                iounmap(ndev->mw[i].vbase);
1424
1425        iounmap(ndev->reg_base);
1426        pci_release_selected_regions(pdev, NTB_BAR_MASK);
1427        pci_disable_device(pdev);
1428        ntb_free_debugfs(ndev);
1429        kfree(ndev);
1430}
1431
1432static struct pci_driver ntb_pci_driver = {
1433        .name = KBUILD_MODNAME,
1434        .id_table = ntb_pci_tbl,
1435        .probe = ntb_pci_probe,
1436        .remove = ntb_pci_remove,
1437};
1438module_pci_driver(ntb_pci_driver);
1439
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.