/* bnx2x_main.c: Broadcom Everest network driver.
 *
 * Copyright (c) 2007-2008 Broadcom Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation.
 *
 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
 * Written by: Eliezer Tamir
 * Based on code from Michael Chan's bnx2 driver
 * UDP CSUM errata workaround by Arik Gendelman
 * Slowpath rework by Vladislav Zolotarov
 * Statistics and Link management by Yitchak Gertner
 *
 */

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h>
#include <linux/device.h>  /* for dev_info() */
#include <linux/timer.h>
#include <linux/errno.h>
#include <linux/ioport.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/dma-mapping.h>
#include <linux/bitops.h>
#include <linux/irq.h>
#include <linux/delay.h>
#include <asm/byteorder.h>
#include <linux/time.h>
#include <linux/ethtool.h>
#include <linux/mii.h>
#ifdef NETIF_F_HW_VLAN_TX
        #include <linux/if_vlan.h>
#endif
#include <net/ip.h>
#include <net/tcp.h>
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/workqueue.h>
#include <linux/crc32.h>
#include <linux/crc32c.h>
#include <linux/prefetch.h>
#include <linux/zlib.h>
#include <linux/io.h>

#include "bnx2x_reg.h"
#include "bnx2x_fw_defs.h"
#include "bnx2x_hsi.h"
#include "bnx2x_link.h"
#include "bnx2x.h"
#include "bnx2x_init.h"

#define DRV_MODULE_VERSION      "1.45.23"
#define DRV_MODULE_RELDATE      "2008/11/03"
#define BNX2X_BC_VER            0x040200

/* Time in jiffies before concluding the transmitter is hung */
#define TX_TIMEOUT              (5*HZ)

static char version[] __devinitdata =
        "Broadcom NetXtreme II 5771x 10Gigabit Ethernet Driver "
        DRV_MODULE_NAME " " DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

MODULE_AUTHOR("Eliezer Tamir");
MODULE_DESCRIPTION("Broadcom NetXtreme II BCM57710 Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_MODULE_VERSION);

static int disable_tpa;
static int use_inta;
static int poll;
static int debug;
static int load_count[3]; /* 0-common, 1-port0, 2-port1 */
static int use_multi;

module_param(disable_tpa, int, 0);
module_param(use_inta, int, 0);
module_param(poll, int, 0);
module_param(debug, int, 0);
MODULE_PARM_DESC(disable_tpa, "disable the TPA (LRO) feature");
MODULE_PARM_DESC(use_inta, "use INT#A instead of MSI-X");
MODULE_PARM_DESC(poll, "use polling (for debug)");
MODULE_PARM_DESC(debug, "default debug msglevel");

#ifdef BNX2X_MULTI
module_param(use_multi, int, 0);
MODULE_PARM_DESC(use_multi, "use per-CPU queues");
#endif

enum bnx2x_board_type {
        BCM57710 = 0,
        BCM57711 = 1,
        BCM57711E = 2,
};

/* indexed by board_type, above */
static struct {
        char *name;
} board_info[] __devinitdata = {
        { "Broadcom NetXtreme II BCM57710 XGb" },
        { "Broadcom NetXtreme II BCM57711 XGb" },
        { "Broadcom NetXtreme II BCM57711E XGb" }
};


static const struct pci_device_id bnx2x_pci_tbl[] = {
        { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_57710,
                PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM57710 },
        { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_57711,
                PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM57711 },
        { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_57711E,
                PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM57711E },
        { 0 }
};

MODULE_DEVICE_TABLE(pci, bnx2x_pci_tbl);

/****************************************************************************
* General service functions
****************************************************************************/

/* used only at init
 * locking is done by mcp
 */
static void bnx2x_reg_wr_ind(struct bnx2x *bp, u32 addr, u32 val)
{
        pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS, addr);
        pci_write_config_dword(bp->pdev, PCICFG_GRC_DATA, val);
        pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS,
                               PCICFG_VENDOR_ID_OFFSET);
}

static u32 bnx2x_reg_rd_ind(struct bnx2x *bp, u32 addr)
{
        u32 val;

        pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS, addr);
        pci_read_config_dword(bp->pdev, PCICFG_GRC_DATA, &val);
        pci_write_config_dword(bp->pdev, PCICFG_GRC_ADDRESS,
                               PCICFG_VENDOR_ID_OFFSET);

        return val;
}

static const u32 dmae_reg_go_c[] = {
        DMAE_REG_GO_C0, DMAE_REG_GO_C1, DMAE_REG_GO_C2, DMAE_REG_GO_C3,
        DMAE_REG_GO_C4, DMAE_REG_GO_C5, DMAE_REG_GO_C6, DMAE_REG_GO_C7,
        DMAE_REG_GO_C8, DMAE_REG_GO_C9, DMAE_REG_GO_C10, DMAE_REG_GO_C11,
        DMAE_REG_GO_C12, DMAE_REG_GO_C13, DMAE_REG_GO_C14, DMAE_REG_GO_C15
};

/* copy command into DMAE command memory and set DMAE command go */
static void bnx2x_post_dmae(struct bnx2x *bp, struct dmae_command *dmae,
                            int idx)
{
        u32 cmd_offset;
        int i;

        cmd_offset = (DMAE_REG_CMD_MEM + sizeof(struct dmae_command) * idx);
        for (i = 0; i < (sizeof(struct dmae_command)/4); i++) {
                REG_WR(bp, cmd_offset + i*4, *(((u32 *)dmae) + i));

                DP(BNX2X_MSG_OFF, "DMAE cmd[%d].%d (0x%08x) : 0x%08x\n",
                   idx, i, cmd_offset + i*4, *(((u32 *)dmae) + i));
        }
        REG_WR(bp, dmae_reg_go_c[idx], 1);
}

void bnx2x_write_dmae(struct bnx2x *bp, dma_addr_t dma_addr, u32 dst_addr,
                      u32 len32)
{
        struct dmae_command *dmae = &bp->init_dmae;
        u32 *wb_comp = bnx2x_sp(bp, wb_comp);
        int cnt = 200;

        if (!bp->dmae_ready) {
                u32 *data = bnx2x_sp(bp, wb_data[0]);

                DP(BNX2X_MSG_OFF, "DMAE is not ready (dst_addr %08x  len32 %d)"
                   "  using indirect\n", dst_addr, len32);
                bnx2x_init_ind_wr(bp, dst_addr, data, len32);
                return;
        }

        mutex_lock(&bp->dmae_mutex);

        memset(dmae, 0, sizeof(struct dmae_command));

        dmae->opcode = (DMAE_CMD_SRC_PCI | DMAE_CMD_DST_GRC |
                        DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
                        DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
#ifdef __BIG_ENDIAN
                        DMAE_CMD_ENDIANITY_B_DW_SWAP |
#else
                        DMAE_CMD_ENDIANITY_DW_SWAP |
#endif
                        (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
                        (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
        dmae->src_addr_lo = U64_LO(dma_addr);
        dmae->src_addr_hi = U64_HI(dma_addr);
        dmae->dst_addr_lo = dst_addr >> 2;
        dmae->dst_addr_hi = 0;
        dmae->len = len32;
        dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, wb_comp));
        dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, wb_comp));
        dmae->comp_val = DMAE_COMP_VAL;

        DP(BNX2X_MSG_OFF, "dmae: opcode 0x%08x\n"
           DP_LEVEL "src_addr  [%x:%08x]  len [%d *4]  "
                    "dst_addr [%x:%08x (%08x)]\n"
           DP_LEVEL "comp_addr [%x:%08x]  comp_val 0x%08x\n",
           dmae->opcode, dmae->src_addr_hi, dmae->src_addr_lo,
           dmae->len, dmae->dst_addr_hi, dmae->dst_addr_lo, dst_addr,
           dmae->comp_addr_hi, dmae->comp_addr_lo, dmae->comp_val);
        DP(BNX2X_MSG_OFF, "data [0x%08x 0x%08x 0x%08x 0x%08x]\n",
           bp->slowpath->wb_data[0], bp->slowpath->wb_data[1],
           bp->slowpath->wb_data[2], bp->slowpath->wb_data[3]);

        *wb_comp = 0;

        bnx2x_post_dmae(bp, dmae, INIT_DMAE_C(bp));

        udelay(5);

        while (*wb_comp != DMAE_COMP_VAL) {
                DP(BNX2X_MSG_OFF, "wb_comp 0x%08x\n", *wb_comp);

                if (!cnt) {
                        BNX2X_ERR("dmae timeout!\n");
                        break;
                }
                cnt--;
                /* adjust delay for emulation/FPGA */
                if (CHIP_REV_IS_SLOW(bp))
                        msleep(100);
                else
                        udelay(5);
        }

        mutex_unlock(&bp->dmae_mutex);
}

void bnx2x_read_dmae(struct bnx2x *bp, u32 src_addr, u32 len32)
{
        struct dmae_command *dmae = &bp->init_dmae;
        u32 *wb_comp = bnx2x_sp(bp, wb_comp);
        int cnt = 200;

        if (!bp->dmae_ready) {
                u32 *data = bnx2x_sp(bp, wb_data[0]);
                int i;

                DP(BNX2X_MSG_OFF, "DMAE is not ready (src_addr %08x  len32 %d)"
                   "  using indirect\n", src_addr, len32);
                for (i = 0; i < len32; i++)
                        data[i] = bnx2x_reg_rd_ind(bp, src_addr + i*4);
                return;
        }

        mutex_lock(&bp->dmae_mutex);

        memset(bnx2x_sp(bp, wb_data[0]), 0, sizeof(u32) * 4);
        memset(dmae, 0, sizeof(struct dmae_command));

        dmae->opcode = (DMAE_CMD_SRC_GRC | DMAE_CMD_DST_PCI |
                        DMAE_CMD_C_DST_PCI | DMAE_CMD_C_ENABLE |
                        DMAE_CMD_SRC_RESET | DMAE_CMD_DST_RESET |
#ifdef __BIG_ENDIAN
                        DMAE_CMD_ENDIANITY_B_DW_SWAP |
#else
                        DMAE_CMD_ENDIANITY_DW_SWAP |
#endif
                        (BP_PORT(bp) ? DMAE_CMD_PORT_1 : DMAE_CMD_PORT_0) |
                        (BP_E1HVN(bp) << DMAE_CMD_E1HVN_SHIFT));
        dmae->src_addr_lo = src_addr >> 2;
        dmae->src_addr_hi = 0;
        dmae->dst_addr_lo = U64_LO(bnx2x_sp_mapping(bp, wb_data));
        dmae->dst_addr_hi = U64_HI(bnx2x_sp_mapping(bp, wb_data));
        dmae->len = len32;
        dmae->comp_addr_lo = U64_LO(bnx2x_sp_mapping(bp, wb_comp));
        dmae->comp_addr_hi = U64_HI(bnx2x_sp_mapping(bp, wb_comp));
        dmae->comp_val = DMAE_COMP_VAL;

        DP(BNX2X_MSG_OFF, "dmae: opcode 0x%08x\n"
           DP_LEVEL "src_addr  [%x:%08x]  len [%d *4]  "
                    "dst_addr [%x:%08x (%08x)]\n"
           DP_LEVEL "comp_addr [%x:%08x]  comp_val 0x%08x\n",
           dmae->opcode, dmae->src_addr_hi, dmae->src_addr_lo,
           dmae->len, dmae->dst_addr_hi, dmae->dst_addr_lo, src_addr,
           dmae->comp_addr_hi, dmae->comp_addr_lo, dmae->comp_val);

        *wb_comp = 0;

        bnx2x_post_dmae(bp, dmae, INIT_DMAE_C(bp));

        udelay(5);

        while (*wb_comp != DMAE_COMP_VAL) {

                if (!cnt) {
                        BNX2X_ERR("dmae timeout!\n");
                        break;
                }
                cnt--;
                /* adjust delay for emulation/FPGA */
                if (CHIP_REV_IS_SLOW(bp))
                        msleep(100);
                else
                        udelay(5);
        }
        DP(BNX2X_MSG_OFF, "data [0x%08x 0x%08x 0x%08x 0x%08x]\n",
           bp->slowpath->wb_data[0], bp->slowpath->wb_data[1],
           bp->slowpath->wb_data[2], bp->slowpath->wb_data[3]);

        mutex_unlock(&bp->dmae_mutex);
}

/* used only for slowpath so not inlined */
static void bnx2x_wb_wr(struct bnx2x *bp, int reg, u32 val_hi, u32 val_lo)
{
        u32 wb_write[2];

        wb_write[0] = val_hi;
        wb_write[1] = val_lo;
        REG_WR_DMAE(bp, reg, wb_write, 2);
}

#ifdef USE_WB_RD
static u64 bnx2x_wb_rd(struct bnx2x *bp, int reg)
{
        u32 wb_data[2];

        REG_RD_DMAE(bp, reg, wb_data, 2);

        return HILO_U64(wb_data[0], wb_data[1]);
}
#endif

static int bnx2x_mc_assert(struct bnx2x *bp)
{
        char last_idx;
        int i, rc = 0;
        u32 row0, row1, row2, row3;

        /* XSTORM */
        last_idx = REG_RD8(bp, BAR_XSTRORM_INTMEM +
                           XSTORM_ASSERT_LIST_INDEX_OFFSET);
        if (last_idx)
                BNX2X_ERR("XSTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);

        /* print the asserts */
        for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {

                row0 = REG_RD(bp, BAR_XSTRORM_INTMEM +
                              XSTORM_ASSERT_LIST_OFFSET(i));
                row1 = REG_RD(bp, BAR_XSTRORM_INTMEM +
                              XSTORM_ASSERT_LIST_OFFSET(i) + 4);
                row2 = REG_RD(bp, BAR_XSTRORM_INTMEM +
                              XSTORM_ASSERT_LIST_OFFSET(i) + 8);
                row3 = REG_RD(bp, BAR_XSTRORM_INTMEM +
                              XSTORM_ASSERT_LIST_OFFSET(i) + 12);

                if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
                        BNX2X_ERR("XSTORM_ASSERT_INDEX 0x%x = 0x%08x"
                                  " 0x%08x 0x%08x 0x%08x\n",
                                  i, row3, row2, row1, row0);
                        rc++;
                } else {
                        break;
                }
        }

        /* TSTORM */
        last_idx = REG_RD8(bp, BAR_TSTRORM_INTMEM +
                           TSTORM_ASSERT_LIST_INDEX_OFFSET);
        if (last_idx)
                BNX2X_ERR("TSTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);

        /* print the asserts */
        for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {

                row0 = REG_RD(bp, BAR_TSTRORM_INTMEM +
                              TSTORM_ASSERT_LIST_OFFSET(i));
                row1 = REG_RD(bp, BAR_TSTRORM_INTMEM +
                              TSTORM_ASSERT_LIST_OFFSET(i) + 4);
                row2 = REG_RD(bp, BAR_TSTRORM_INTMEM +
                              TSTORM_ASSERT_LIST_OFFSET(i) + 8);
                row3 = REG_RD(bp, BAR_TSTRORM_INTMEM +
                              TSTORM_ASSERT_LIST_OFFSET(i) + 12);

                if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
                        BNX2X_ERR("TSTORM_ASSERT_INDEX 0x%x = 0x%08x"
                                  " 0x%08x 0x%08x 0x%08x\n",
                                  i, row3, row2, row1, row0);
                        rc++;
                } else {
                        break;
                }
        }

        /* CSTORM */
        last_idx = REG_RD8(bp, BAR_CSTRORM_INTMEM +
                           CSTORM_ASSERT_LIST_INDEX_OFFSET);
        if (last_idx)
                BNX2X_ERR("CSTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);

        /* print the asserts */
        for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {

                row0 = REG_RD(bp, BAR_CSTRORM_INTMEM +
                              CSTORM_ASSERT_LIST_OFFSET(i));
                row1 = REG_RD(bp, BAR_CSTRORM_INTMEM +
                              CSTORM_ASSERT_LIST_OFFSET(i) + 4);
                row2 = REG_RD(bp, BAR_CSTRORM_INTMEM +
                              CSTORM_ASSERT_LIST_OFFSET(i) + 8);
                row3 = REG_RD(bp, BAR_CSTRORM_INTMEM +
                              CSTORM_ASSERT_LIST_OFFSET(i) + 12);

                if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
                        BNX2X_ERR("CSTORM_ASSERT_INDEX 0x%x = 0x%08x"
                                  " 0x%08x 0x%08x 0x%08x\n",
                                  i, row3, row2, row1, row0);
                        rc++;
                } else {
                        break;
                }
        }

        /* USTORM */
        last_idx = REG_RD8(bp, BAR_USTRORM_INTMEM +
                           USTORM_ASSERT_LIST_INDEX_OFFSET);
        if (last_idx)
                BNX2X_ERR("USTORM_ASSERT_LIST_INDEX 0x%x\n", last_idx);

        /* print the asserts */
        for (i = 0; i < STROM_ASSERT_ARRAY_SIZE; i++) {

                row0 = REG_RD(bp, BAR_USTRORM_INTMEM +
                              USTORM_ASSERT_LIST_OFFSET(i));
                row1 = REG_RD(bp, BAR_USTRORM_INTMEM +
                              USTORM_ASSERT_LIST_OFFSET(i) + 4);
                row2 = REG_RD(bp, BAR_USTRORM_INTMEM +
                              USTORM_ASSERT_LIST_OFFSET(i) + 8);
                row3 = REG_RD(bp, BAR_USTRORM_INTMEM +
                              USTORM_ASSERT_LIST_OFFSET(i) + 12);

                if (row0 != COMMON_ASM_INVALID_ASSERT_OPCODE) {
                        BNX2X_ERR("USTORM_ASSERT_INDEX 0x%x = 0x%08x"
                                  " 0x%08x 0x%08x 0x%08x\n",
                                  i, row3, row2, row1, row0);
                        rc++;
                } else {
                        break;
                }
        }

        return rc;
}

static void bnx2x_fw_dump(struct bnx2x *bp)
{
        u32 mark, offset;
        u32 data[9];
        int word;

        mark = REG_RD(bp, MCP_REG_MCPR_SCRATCH + 0xf104);
        mark = ((mark + 0x3) & ~0x3);
        printk(KERN_ERR PFX "begin fw dump (mark 0x%x)\n" KERN_ERR, mark);

        for (offset = mark - 0x08000000; offset <= 0xF900; offset += 0x8*4) {
                for (word = 0; word < 8; word++)
                        data[word] = htonl(REG_RD(bp, MCP_REG_MCPR_SCRATCH +
                                                  offset + 4*word));
                data[8] = 0x0;
                printk(KERN_CONT "%s", (char *)data);
        }
        for (offset = 0xF108; offset <= mark - 0x08000000; offset += 0x8*4) {
                for (word = 0; word < 8; word++)
                        data[word] = htonl(REG_RD(bp, MCP_REG_MCPR_SCRATCH +
                                                  offset + 4*word));
                data[8] = 0x0;
                printk(KERN_CONT "%s", (char *)data);
        }
        printk("\n" KERN_ERR PFX "end of fw dump\n");
}

static void bnx2x_panic_dump(struct bnx2x *bp)
{
        int i;
        u16 j, start, end;

        bp->stats_state = STATS_STATE_DISABLED;
        DP(BNX2X_MSG_STATS, "stats_state - DISABLED\n");

        BNX2X_ERR("begin crash dump -----------------\n");

        for_each_queue(bp, i) {
                struct bnx2x_fastpath *fp = &bp->fp[i];
                struct eth_tx_db_data *hw_prods = fp->hw_tx_prods;

                BNX2X_ERR("queue[%d]: tx_pkt_prod(%x)  tx_pkt_cons(%x)"
                          "  tx_bd_prod(%x)  tx_bd_cons(%x)  *tx_cons_sb(%x)\n",
                          i, fp->tx_pkt_prod, fp->tx_pkt_cons, fp->tx_bd_prod,
                          fp->tx_bd_cons, le16_to_cpu(*fp->tx_cons_sb));
                BNX2X_ERR("          rx_bd_prod(%x)  rx_bd_cons(%x)"
                          "  *rx_bd_cons_sb(%x)  rx_comp_prod(%x)"
                          "  rx_comp_cons(%x)  *rx_cons_sb(%x)\n",
                          fp->rx_bd_prod, fp->rx_bd_cons,
                          le16_to_cpu(*fp->rx_bd_cons_sb), fp->rx_comp_prod,
                          fp->rx_comp_cons, le16_to_cpu(*fp->rx_cons_sb));
                BNX2X_ERR("          rx_sge_prod(%x)  last_max_sge(%x)"
                          "  fp_c_idx(%x)  *sb_c_idx(%x)  fp_u_idx(%x)"
                          "  *sb_u_idx(%x)  bd data(%x,%x)\n",
                          fp->rx_sge_prod, fp->last_max_sge, fp->fp_c_idx,
                          fp->status_blk->c_status_block.status_block_index,
                          fp->fp_u_idx,
                          fp->status_blk->u_status_block.status_block_index,
                          hw_prods->packets_prod, hw_prods->bds_prod);

                start = TX_BD(le16_to_cpu(*fp->tx_cons_sb) - 10);
                end = TX_BD(le16_to_cpu(*fp->tx_cons_sb) + 245);
                for (j = start; j < end; j++) {
                        struct sw_tx_bd *sw_bd = &fp->tx_buf_ring[j];

                        BNX2X_ERR("packet[%x]=[%p,%x]\n", j,
                                  sw_bd->skb, sw_bd->first_bd);
                }

                start = TX_BD(fp->tx_bd_cons - 10);
                end = TX_BD(fp->tx_bd_cons + 254);
                for (j = start; j < end; j++) {
                        u32 *tx_bd = (u32 *)&fp->tx_desc_ring[j];

                        BNX2X_ERR("tx_bd[%x]=[%x:%x:%x:%x]\n",
                                  j, tx_bd[0], tx_bd[1], tx_bd[2], tx_bd[3]);
                }

                start = RX_BD(le16_to_cpu(*fp->rx_cons_sb) - 10);
                end = RX_BD(le16_to_cpu(*fp->rx_cons_sb) + 503);
                for (j = start; j < end; j++) {
                        u32 *rx_bd = (u32 *)&fp->rx_desc_ring[j];
                        struct sw_rx_bd *sw_bd = &fp->rx_buf_ring[j];

                        BNX2X_ERR("rx_bd[%x]=[%x:%x]  sw_bd=[%p]\n",
                                  j, rx_bd[1], rx_bd[0], sw_bd->skb);
                }

                start = RX_SGE(fp->rx_sge_prod);
                end = RX_SGE(fp->last_max_sge);
                for (j = start; j < end; j++) {
                        u32 *rx_sge = (u32 *)&fp->rx_sge_ring[j];
                        struct sw_rx_page *sw_page = &fp->rx_page_ring[j];

                        BNX2X_ERR("rx_sge[%x]=[%x:%x]  sw_page=[%p]\n",
                                  j, rx_sge[1], rx_sge[0], sw_page->page);
                }

                start = RCQ_BD(fp->rx_comp_cons - 10);
                end = RCQ_BD(fp->rx_comp_cons + 503);
                for (j = start; j < end; j++) {
                        u32 *cqe = (u32 *)&fp->rx_comp_ring[j];

                        BNX2X_ERR("cqe[%x]=[%x:%x:%x:%x]\n",
                                  j, cqe[0], cqe[1], cqe[2], cqe[3]);
                }
        }

        BNX2X_ERR("def_c_idx(%u)  def_u_idx(%u)  def_x_idx(%u)"
                  "  def_t_idx(%u)  def_att_idx(%u)  attn_state(%u)"
                  "  spq_prod_idx(%u)\n",
                  bp->def_c_idx, bp->def_u_idx, bp->def_x_idx, bp->def_t_idx,
                  bp->def_att_idx, bp->attn_state, bp->spq_prod_idx);

        bnx2x_fw_dump(bp);
        bnx2x_mc_assert(bp);
        BNX2X_ERR("end crash dump -----------------\n");
}

static void bnx2x_int_enable(struct bnx2x *bp)
{
        int port = BP_PORT(bp);
        u32 addr = port ? HC_REG_CONFIG_1 : HC_REG_CONFIG_0;
        u32 val = REG_RD(bp, addr);
        int msix = (bp->flags & USING_MSIX_FLAG) ? 1 : 0;

        if (msix) {
                val &= ~HC_CONFIG_0_REG_SINGLE_ISR_EN_0;
                val |= (HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
                        HC_CONFIG_0_REG_ATTN_BIT_EN_0);
        } else {
                val |= (HC_CONFIG_0_REG_SINGLE_ISR_EN_0 |
                        HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
                        HC_CONFIG_0_REG_INT_LINE_EN_0 |
                        HC_CONFIG_0_REG_ATTN_BIT_EN_0);

                DP(NETIF_MSG_INTR, "write %x to HC %d (addr 0x%x)  MSI-X %d\n",
                   val, port, addr, msix);

                REG_WR(bp, addr, val);

                val &= ~HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0;
        }

        DP(NETIF_MSG_INTR, "write %x to HC %d (addr 0x%x)  MSI-X %d\n",
           val, port, addr, msix);

        REG_WR(bp, addr, val);

        if (CHIP_IS_E1H(bp)) {
                /* init leading/trailing edge */
                if (IS_E1HMF(bp)) {
                        val = (0xfe0f | (1 << (BP_E1HVN(bp) + 4)));
                        if (bp->port.pmf)
                                /* enable nig attention */
                                val |= 0x0100;
                } else
                        val = 0xffff;

                REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, val);
                REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, val);
        }
}

static void bnx2x_int_disable(struct bnx2x *bp)
{
        int port = BP_PORT(bp);
        u32 addr = port ? HC_REG_CONFIG_1 : HC_REG_CONFIG_0;
        u32 val = REG_RD(bp, addr);

        val &= ~(HC_CONFIG_0_REG_SINGLE_ISR_EN_0 |
                 HC_CONFIG_0_REG_MSI_MSIX_INT_EN_0 |
                 HC_CONFIG_0_REG_INT_LINE_EN_0 |
                 HC_CONFIG_0_REG_ATTN_BIT_EN_0);

        DP(NETIF_MSG_INTR, "write %x to HC %d (addr 0x%x)\n",
           val, port, addr);

        REG_WR(bp, addr, val);
        if (REG_RD(bp, addr) != val)
                BNX2X_ERR("BUG! proper val not read from IGU!\n");
}

static void bnx2x_int_disable_sync(struct bnx2x *bp, int disable_hw)
{
        int msix = (bp->flags & USING_MSIX_FLAG) ? 1 : 0;
        int i;

        /* disable interrupt handling */
        atomic_inc(&bp->intr_sem);
        if (disable_hw)
                /* prevent the HW from sending interrupts */
                bnx2x_int_disable(bp);

        /* make sure all ISRs are done */
        if (msix) {
                for_each_queue(bp, i)
                        synchronize_irq(bp->msix_table[i].vector);

                /* one more for the Slow Path IRQ */
                synchronize_irq(bp->msix_table[i].vector);
        } else
                synchronize_irq(bp->pdev->irq);

        /* make sure sp_task is not running */
        cancel_work_sync(&bp->sp_task);
}

/* fast path */

/*
 * General service functions
 */

static inline void bnx2x_ack_sb(struct bnx2x *bp, u8 sb_id,
                                u8 storm, u16 index, u8 op, u8 update)
{
        u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp)*32 +
                       COMMAND_REG_INT_ACK);
        struct igu_ack_register igu_ack;

        igu_ack.status_block_index = index;
        igu_ack.sb_id_and_flags =
                        ((sb_id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) |
                         (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) |
                         (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) |
                         (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT));

        DP(BNX2X_MSG_OFF, "write 0x%08x to HC addr 0x%x\n",
           (*(u32 *)&igu_ack), hc_addr);
        REG_WR(bp, hc_addr, (*(u32 *)&igu_ack));
}

static inline u16 bnx2x_update_fpsb_idx(struct bnx2x_fastpath *fp)
{
        struct host_status_block *fpsb = fp->status_blk;
        u16 rc = 0;

        barrier(); /* status block is written to by the chip */
        if (fp->fp_c_idx != fpsb->c_status_block.status_block_index) {
                fp->fp_c_idx = fpsb->c_status_block.status_block_index;
                rc |= 1;
        }
        if (fp->fp_u_idx != fpsb->u_status_block.status_block_index) {
                fp->fp_u_idx = fpsb->u_status_block.status_block_index;
                rc |= 2;
        }
        return rc;
}

static u16 bnx2x_ack_int(struct bnx2x *bp)
{
        u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp)*32 +
                       COMMAND_REG_SIMD_MASK);
        u32 result = REG_RD(bp, hc_addr);

        DP(BNX2X_MSG_OFF, "read 0x%08x from HC addr 0x%x\n",
           result, hc_addr);

        return result;
}


/*
 * fast path service functions
 */

/* free skb in the packet ring at pos idx
 * return idx of last bd freed
 */
static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
                             u16 idx)
{
        struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
        struct eth_tx_bd *tx_bd;
        struct sk_buff *skb = tx_buf->skb;
        u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
        int nbd;

        DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
           idx, tx_buf, skb);

        /* unmap first bd */
        DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
        tx_bd = &fp->tx_desc_ring[bd_idx];
        pci_unmap_single(bp->pdev, BD_UNMAP_ADDR(tx_bd),
                         BD_UNMAP_LEN(tx_bd), PCI_DMA_TODEVICE);

        nbd = le16_to_cpu(tx_bd->nbd) - 1;
        new_cons = nbd + tx_buf->first_bd;
#ifdef BNX2X_STOP_ON_ERROR
        if (nbd > (MAX_SKB_FRAGS + 2)) {
                BNX2X_ERR("BAD nbd!\n");
                bnx2x_panic();
        }
#endif

        /* Skip a parse bd and the TSO split header bd
           since they have no mapping */
        if (nbd)
                bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));

        if (tx_bd->bd_flags.as_bitfield & (ETH_TX_BD_FLAGS_IP_CSUM |
                                           ETH_TX_BD_FLAGS_TCP_CSUM |
                                           ETH_TX_BD_FLAGS_SW_LSO)) {
                if (--nbd)
                        bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
                tx_bd = &fp->tx_desc_ring[bd_idx];
                /* is this a TSO split header bd? */
                if (tx_bd->bd_flags.as_bitfield & ETH_TX_BD_FLAGS_SW_LSO) {
                        if (--nbd)
                                bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
                }
        }

        /* now free frags */
        while (nbd > 0) {

                DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
                tx_bd = &fp->tx_desc_ring[bd_idx];
                pci_unmap_page(bp->pdev, BD_UNMAP_ADDR(tx_bd),
                               BD_UNMAP_LEN(tx_bd), PCI_DMA_TODEVICE);
                if (--nbd)
                        bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
        }

        /* release skb */
        WARN_ON(!skb);
        dev_kfree_skb(skb);
        tx_buf->first_bd = 0;
        tx_buf->skb = NULL;

        return new_cons;
}

static inline u16 bnx2x_tx_avail(struct bnx2x_fastpath *fp)
{
        s16 used;
        u16 prod;
        u16 cons;

        barrier(); /* Tell compiler that prod and cons can change */
        prod = fp->tx_bd_prod;
        cons = fp->tx_bd_cons;

        /* NUM_TX_RINGS = number of "next-page" entries
           It will be used as a threshold */
        used = SUB_S16(prod, cons) + (s16)NUM_TX_RINGS;

#ifdef BNX2X_STOP_ON_ERROR
        WARN_ON(used < 0);
        WARN_ON(used > fp->bp->tx_ring_size);
        WARN_ON((fp->bp->tx_ring_size - used) > MAX_TX_AVAIL);
#endif

        return (s16)(fp->bp->tx_ring_size) - used;
}

static void bnx2x_tx_int(struct bnx2x_fastpath *fp, int work)
{
        struct bnx2x *bp = fp->bp;
        u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
        int done = 0;

#ifdef BNX2X_STOP_ON_ERROR
        if (unlikely(bp->panic))
                return;
#endif

        hw_cons = le16_to_cpu(*fp->tx_cons_sb);
        sw_cons = fp->tx_pkt_cons;

        while (sw_cons != hw_cons) {
                u16 pkt_cons;

                pkt_cons = TX_BD(sw_cons);

                /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */

                DP(NETIF_MSG_TX_DONE, "hw_cons %u  sw_cons %u  pkt_cons %u\n",
                   hw_cons, sw_cons, pkt_cons);

/*              if (NEXT_TX_IDX(sw_cons) != hw_cons) {
                        rmb();
                        prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
                }
*/
                bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
                sw_cons++;
                done++;

                if (done == work)
                        break;
        }

        fp->tx_pkt_cons = sw_cons;
        fp->tx_bd_cons = bd_cons;

        /* Need to make the tx_cons update visible to start_xmit()
         * before checking for netif_queue_stopped().  Without the
         * memory barrier, there is a small possibility that start_xmit()
         * will miss it and cause the queue to be stopped forever.
         */
        smp_mb();

        /* TBD need a thresh? */
        if (unlikely(netif_queue_stopped(bp->dev))) {

                netif_tx_lock(bp->dev);

                if (netif_queue_stopped(bp->dev) &&
                    (bp->state == BNX2X_STATE_OPEN) &&
                    (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
                        netif_wake_queue(bp->dev);

                netif_tx_unlock(bp->dev);
        }
}


static void bnx2x_sp_event(struct bnx2x_fastpath *fp,
                           union eth_rx_cqe *rr_cqe)
{
        struct bnx2x *bp = fp->bp;
        int cid = SW_CID(rr_cqe->ramrod_cqe.conn_and_cmd_data);
        int command = CQE_CMD(rr_cqe->ramrod_cqe.conn_and_cmd_data);

        DP(BNX2X_MSG_SP,
           "fp %d  cid %d  got ramrod #%d  state is %x  type is %d\n",
           FP_IDX(fp), cid, command, bp->state,
           rr_cqe->ramrod_cqe.ramrod_type);

        bp->spq_left++;

        if (FP_IDX(fp)) {
                switch (command | fp->state) {
                case (RAMROD_CMD_ID_ETH_CLIENT_SETUP |
                                                BNX2X_FP_STATE_OPENING):
                        DP(NETIF_MSG_IFUP, "got MULTI[%d] setup ramrod\n",
                           cid);
                        fp->state = BNX2X_FP_STATE_OPEN;
                        break;

                case (RAMROD_CMD_ID_ETH_HALT | BNX2X_FP_STATE_HALTING):
                        DP(NETIF_MSG_IFDOWN, "got MULTI[%d] halt ramrod\n",
                           cid);
                        fp->state = BNX2X_FP_STATE_HALTED;
                        break;

                default:
                        BNX2X_ERR("unexpected MC reply (%d)  "
                                  "fp->state is %x\n", command, fp->state);
                        break;
                }
                mb(); /* force bnx2x_wait_ramrod() to see the change */
                return;
        }

        switch (command | bp->state) {
        case (RAMROD_CMD_ID_ETH_PORT_SETUP | BNX2X_STATE_OPENING_WAIT4_PORT):
                DP(NETIF_MSG_IFUP, "got setup ramrod\n");
                bp->state = BNX2X_STATE_OPEN;
                break;

        case (RAMROD_CMD_ID_ETH_HALT | BNX2X_STATE_CLOSING_WAIT4_HALT):
                DP(NETIF_MSG_IFDOWN, "got halt ramrod\n");
                bp->state = BNX2X_STATE_CLOSING_WAIT4_DELETE;
                fp->state = BNX2X_FP_STATE_HALTED;
                break;

        case (RAMROD_CMD_ID_ETH_CFC_DEL | BNX2X_STATE_CLOSING_WAIT4_HALT):
                DP(NETIF_MSG_IFDOWN, "got delete ramrod for MULTI[%d]\n", cid);
                bnx2x_fp(bp, cid, state) = BNX2X_FP_STATE_CLOSED;
                break;


        case (RAMROD_CMD_ID_ETH_SET_MAC | BNX2X_STATE_OPEN):
        case (RAMROD_CMD_ID_ETH_SET_MAC | BNX2X_STATE_DIAG):
                DP(NETIF_MSG_IFUP, "got set mac ramrod\n");
                bp->set_mac_pending = 0;
                break;

        case (RAMROD_CMD_ID_ETH_SET_MAC | BNX2X_STATE_CLOSING_WAIT4_HALT):
                DP(NETIF_MSG_IFDOWN, "got (un)set mac ramrod\n");
                break;

        default:
                BNX2X_ERR("unexpected MC reply (%d)  bp->state is %x\n",
                          command, bp->state);
                break;
        }
        mb(); /* force bnx2x_wait_ramrod() to see the change */
}

static inline void bnx2x_free_rx_sge(struct bnx2x *bp,
                                     struct bnx2x_fastpath *fp, u16 index)
{
        struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
        struct page *page = sw_buf->page;
        struct eth_rx_sge *sge = &fp->rx_sge_ring[index];

        /* Skip "next page" elements */
        if (!page)
                return;

        pci_unmap_page(bp->pdev, pci_unmap_addr(sw_buf, mapping),
                       BCM_PAGE_SIZE*PAGES_PER_SGE, PCI_DMA_FROMDEVICE);
        __free_pages(page, PAGES_PER_SGE_SHIFT);

        sw_buf->page = NULL;
        sge->addr_hi = 0;
        sge->addr_lo = 0;
}

static inline void bnx2x_free_rx_sge_range(struct bnx2x *bp,
                                           struct bnx2x_fastpath *fp, int last)
{
        int i;

        for (i = 0; i < last; i++)
                bnx2x_free_rx_sge(bp, fp, i);
}

static inline int bnx2x_alloc_rx_sge(struct bnx2x *bp,
                                     struct bnx2x_fastpath *fp, u16 index)
{
        struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
        struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
        struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
        dma_addr_t mapping;

        if (unlikely(page == NULL))

                return -ENOMEM;

        mapping = pci_map_page(bp->pdev, page, 0, BCM_PAGE_SIZE*PAGES_PER_SGE,
                               PCI_DMA_FROMDEVICE);
        if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
                __free_pages(page, PAGES_PER_SGE_SHIFT);
                return -ENOMEM;
        }

        sw_buf->page = page;
        pci_unmap_addr_set(sw_buf, mapping, mapping);

        sge->addr_hi = cpu_to_le32(U64_HI(mapping));
        sge->addr_lo = cpu_to_le32(U64_LO(mapping));

        return 0;
}

static inline int bnx2x_alloc_rx_skb(struct bnx2x *bp,
                                     struct bnx2x_fastpath *fp, u16 index)
{
        struct sk_buff *skb;
        struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
        struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
        dma_addr_t mapping;

        skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
        if (unlikely(skb == NULL))
                return -ENOMEM;

        mapping = pci_map_single(bp->pdev, skb->data, bp->rx_buf_size,
                                 PCI_DMA_FROMDEVICE);
        if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
                dev_kfree_skb(skb);
                return -ENOMEM;
        }

        rx_buf->skb = skb;
        pci_unmap_addr_set(rx_buf, mapping, mapping);

        rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
        rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));

        return 0;
}

/* note that we are not allocating a new skb,
 * we are just moving one from cons to prod
 * we are not creating a new mapping,
 * so there is no need to check for dma_mapping_error().
 */
static void bnx2x_reuse_rx_skb(struct bnx2x_fastpath *fp,
                               struct sk_buff *skb, u16 cons, u16 prod)
{
        struct bnx2x *bp = fp->bp;
        struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
        struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
        struct eth_rx_bd *cons_bd = &fp->rx_desc_ring[cons];
        struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];

        pci_dma_sync_single_for_device(bp->pdev,
                                       pci_unmap_addr(cons_rx_buf, mapping),
                                       bp->rx_offset + RX_COPY_THRESH,
                                       PCI_DMA_FROMDEVICE);

        prod_rx_buf->skb = cons_rx_buf->skb;
        pci_unmap_addr_set(prod_rx_buf, mapping,
                           pci_unmap_addr(cons_rx_buf, mapping));
        *prod_bd = *cons_bd;
}

static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
                                             u16 idx)
{
        u16 last_max = fp->last_max_sge;

        if (SUB_S16(idx, last_max) > 0)
                fp->last_max_sge = idx;
}

static void bnx2x_clear_sge_mask_next_elems(struct bnx2x_fastpath *fp)
{
        int i, j;

        for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
                int idx = RX_SGE_CNT * i - 1;

                for (j = 0; j < 2; j++) {
                        SGE_MASK_CLEAR_BIT(fp, idx);
                        idx--;
                }
        }
}

static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
                                  struct eth_fast_path_rx_cqe *fp_cqe)
{
        struct bnx2x *bp = fp->bp;
        u16 sge_len = BCM_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
                                     le16_to_cpu(fp_cqe->len_on_bd)) >>
                      BCM_PAGE_SHIFT;
        u16 last_max, last_elem, first_elem;
        u16 delta = 0;
        u16 i;

        if (!sge_len)
                return;

        /* First mark all used pages */
        for (i = 0; i < sge_len; i++)
                SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));

        DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
           sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));

        /* Here we assume that the last SGE index is the biggest */
        prefetch((void *)(fp->sge_mask));
        bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));

        last_max = RX_SGE(fp->last_max_sge);
        last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
        first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;

        /* If ring is not full */
        if (last_elem + 1 != first_elem)
                last_elem++;

        /* Now update the prod */
        for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
                if (likely(fp->sge_mask[i]))
                        break;

                fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
                delta += RX_SGE_MASK_ELEM_SZ;
        }

        if (delta > 0) {
                fp->rx_sge_prod += delta;
                /* clear page-end entries */
                bnx2x_clear_sge_mask_next_elems(fp);
        }

        DP(NETIF_MSG_RX_STATUS,
           "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
           fp->last_max_sge, fp->rx_sge_prod);
}

static inline void bnx2x_init_sge_ring_bit_mask(struct bnx2x_fastpath *fp)
{
        /* Set the mask to all 1-s: it's faster to compare to 0 than to 0xf-s */
        memset(fp->sge_mask, 0xff,
               (NUM_RX_SGE >> RX_SGE_MASK_ELEM_SHIFT)*sizeof(u64));

        /* Clear the two last indices in the page to 1:
           these are the indices that correspond to the "next" element,
           hence will never be indicated and should be removed from
           the calculations. */
        bnx2x_clear_sge_mask_next_elems(fp);
}

static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
                            struct sk_buff *skb, u16 cons, u16 prod)
{
        struct bnx2x *bp = fp->bp;
        struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
        struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
        struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
        dma_addr_t mapping;

        /* move empty skb from pool to prod and map it */
        prod_rx_buf->skb = fp->tpa_pool[queue].skb;
        mapping = pci_map_single(bp->pdev, fp->tpa_pool[queue].skb->data,
                                 bp->rx_buf_size, PCI_DMA_FROMDEVICE);
        pci_unmap_addr_set(prod_rx_buf, mapping, mapping);

        /* move partial skb from cons to pool (don't unmap yet) */
        fp->tpa_pool[queue] = *cons_rx_buf;

        /* mark bin state as start - print error if current state != stop */
        if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
                BNX2X_ERR("start of bin not in stop [%d]\n", queue);

        fp->tpa_state[queue] = BNX2X_TPA_START;

        /* point prod_bd to new skb */
        prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
        prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));

#ifdef BNX2X_STOP_ON_ERROR
        fp->tpa_queue_used |= (1 << queue);
#ifdef __powerpc64__
        DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
#else
        DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
#endif
           fp->tpa_queue_used);
#endif
}

static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
                               struct sk_buff *skb,
                               struct eth_fast_path_rx_cqe *fp_cqe,
                               u16 cqe_idx)
{
        struct sw_rx_page *rx_pg, old_rx_pg;
        struct page *sge;
        u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
        u32 i, frag_len, frag_size, pages;
        int err;
        int j;

        frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
        pages = BCM_PAGE_ALIGN(frag_size) >> BCM_PAGE_SHIFT;

        /* This is needed in order to enable forwarding support */
        if (frag_size)
                skb_shinfo(skb)->gso_size = min((u32)BCM_PAGE_SIZE,
                                               max(frag_size, (u32)len_on_bd));

#ifdef BNX2X_STOP_ON_ERROR
        if (pages > 8*PAGES_PER_SGE) {
                BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
                          pages, cqe_idx);
                BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
                          fp_cqe->pkt_len, len_on_bd);
                bnx2x_panic();
                return -EINVAL;
        }
#endif

        /* Run through the SGL and compose the fragmented skb */
        for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
                u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));

                /* FW gives the indices of the SGE as if the ring is an array
                   (meaning that "next" element will consume 2 indices) */
                frag_len = min(frag_size, (u32)(BCM_PAGE_SIZE*PAGES_PER_SGE));
                rx_pg = &fp->rx_page_ring[sge_idx];
                sge = rx_pg->page;
                old_rx_pg = *rx_pg;

                /* If we fail to allocate a substitute page, we simply stop
                   where we are and drop the whole packet */
                err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
                if (unlikely(err)) {
                        bp->eth_stats.rx_skb_alloc_failed++;
                        return err;
                }

                /* Unmap the page as we r going to pass it to the stack */
                pci_unmap_page(bp->pdev, pci_unmap_addr(&old_rx_pg, mapping),
                              BCM_PAGE_SIZE*PAGES_PER_SGE, PCI_DMA_FROMDEVICE);

                /* Add one frag and update the appropriate fields in the skb */
                skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);

                skb->data_len += frag_len;
                skb->truesize += frag_len;
                skb->len += frag_len;

                frag_size -= frag_len;
        }

        return 0;
}

static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
                           u16 queue, int pad, int len, union eth_rx_cqe *cqe,
                           u16 cqe_idx)
{
        struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
        struct sk_buff *skb = rx_buf->skb;
        /* alloc new skb */
        struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);

        /* Unmap skb in the pool anyway, as we are going to change
           pool entry status to BNX2X_TPA_STOP even if new skb allocation
           fails. */
        pci_unmap_single(bp->pdev, pci_unmap_addr(rx_buf, mapping),
                         bp->rx_buf_size, PCI_DMA_FROMDEVICE);

        if (likely(new_skb)) {
                /* fix ip xsum and give it to the stack */
                /* (no need to map the new skb) */

                prefetch(skb);
                prefetch(((char *)(skb)) + 128);

#ifdef BNX2X_STOP_ON_ERROR
                if (pad + len > bp->rx_buf_size) {
                        BNX2X_ERR("skb_put is about to fail...  "
                                  "pad %d  len %d  rx_buf_size %d\n",
                                  pad, len, bp->rx_buf_size);
                        bnx2x_panic();
                        return;
                }
#endif

                skb_reserve(skb, pad);
                skb_put(skb, len);

                skb->protocol = eth_type_trans(skb, bp->dev);
                skb->ip_summed = CHECKSUM_UNNECESSARY;

                {
                        struct iphdr *iph;

                        iph = (struct iphdr *)skb->data;
                        iph->check = 0;
                        iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
                }

                if (!bnx2x_fill_frag_skb(bp, fp, skb,
                                         &cqe->fast_path_cqe, cqe_idx)) {
#ifdef BCM_VLAN
                        if ((bp->vlgrp != NULL) &&
                            (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
                             PARSING_FLAGS_VLAN))
                                vlan_hwaccel_receive_skb(skb, bp->vlgrp,
                                                le16_to_cpu(cqe->fast_path_cqe.
                                                            vlan_tag));
                        else
#endif
                                netif_receive_skb(skb);
                } else {
                        DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
                           " - dropping packet!\n");
                        dev_kfree_skb(skb);
                }

                bp->dev->last_rx = jiffies;

                /* put new skb in bin */
                fp->tpa_pool[queue].skb = new_skb;

        } else {
                /* else drop the packet and keep the buffer in the bin */
                DP(NETIF_MSG_RX_STATUS,
                   "Failed to allocate new skb - dropping packet!\n");
                bp->eth_stats.rx_skb_alloc_failed++;
        }

        fp->tpa_state[queue] = BNX2X_TPA_STOP;
}

static inline void bnx2x_update_rx_prod(struct bnx2x *bp,
                                        struct bnx2x_fastpath *fp,
                                        u16 bd_prod, u16 rx_comp_prod,
                                        u16 rx_sge_prod)
{
        struct tstorm_eth_rx_producers rx_prods = {0};
        int i;

        /* Update producers */
        rx_prods.bd_prod = bd_prod;
        rx_prods.cqe_prod = rx_comp_prod;
        rx_prods.sge_prod = rx_sge_prod;

        for (i = 0; i < sizeof(struct tstorm_eth_rx_producers)/4; i++)
                REG_WR(bp, BAR_TSTRORM_INTMEM +
                       TSTORM_RX_PRODS_OFFSET(BP_PORT(bp), FP_CL_ID(fp)) + i*4,
                       ((u32 *)&rx_prods)[i]);

        DP(NETIF_MSG_RX_STATUS,
           "Wrote: bd_prod %u  cqe_prod %u  sge_prod %u\n",
           bd_prod, rx_comp_prod, rx_sge_prod);
}

static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
{
        struct bnx2x *bp = fp->bp;
        u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
        u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
        int rx_pkt = 0;

#ifdef BNX2X_STOP_ON_ERROR
        if (unlikely(bp->panic))
                return 0;
#endif

        /* CQ "next element" is of the size of the regular element,
           that's why it's ok here */
        hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
        if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
                hw_comp_cons++;

        bd_cons = fp->rx_bd_cons;
        bd_prod = fp->rx_bd_prod;
        bd_prod_fw = bd_prod;
        sw_comp_cons = fp->rx_comp_cons;
        sw_comp_prod = fp->rx_comp_prod;

        /* Memory barrier necessary as speculative reads of the rx
         * buffer can be ahead of the index in the status block
         */
        rmb();

        DP(NETIF_MSG_RX_STATUS,
           "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
           FP_IDX(fp), hw_comp_cons, sw_comp_cons);

        while (sw_comp_cons != hw_comp_cons) {
                struct sw_rx_bd *rx_buf = NULL;
                struct sk_buff *skb;
                union eth_rx_cqe *cqe;
                u8 cqe_fp_flags;
                u16 len, pad;

                comp_ring_cons = RCQ_BD(sw_comp_cons);
                bd_prod = RX_BD(bd_prod);
                bd_cons = RX_BD(bd_cons);

                cqe = &fp->rx_comp_ring[comp_ring_cons];
                cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;

                DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
                   "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
                   cqe_fp_flags, cqe->fast_path_cqe.status_flags,
                   cqe->fast_path_cqe.rss_hash_result,
                   le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
                   le16_to_cpu(cqe->fast_path_cqe.pkt_len));

                /* is this a slowpath msg? */
                if (unlikely(CQE_TYPE(cqe_fp_flags))) {
                        bnx2x_sp_event(fp, cqe);
                        goto next_cqe;

                /* this is an rx packet */
                } else {
                        rx_buf = &fp->rx_buf_ring[bd_cons];
                        skb = rx_buf->skb;
                        len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
                        pad = cqe->fast_path_cqe.placement_offset;

                        /* If CQE is marked both TPA_START and TPA_END
                           it is a non-TPA CQE */
                        if ((!fp->disable_tpa) &&
                            (TPA_TYPE(cqe_fp_flags) !=
                                        (TPA_TYPE_START | TPA_TYPE_END))) {
                                u16 queue = cqe->fast_path_cqe.queue_index;

                                if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
                                        DP(NETIF_MSG_RX_STATUS,
                                           "calling tpa_start on queue %d\n",
                                           queue);

                                        bnx2x_tpa_start(fp, queue, skb,
                                                        bd_cons, bd_prod);
                                        goto next_rx;
                                }

                                if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
                                        DP(NETIF_MSG_RX_STATUS,
                                           "calling tpa_stop on queue %d\n",
                                           queue);

                                        if (!BNX2X_RX_SUM_FIX(cqe))
                                                BNX2X_ERR("STOP on none TCP "
                                                          "data\n");

                                        /* This is a size of the linear data
                                           on this skb */
                                        len = le16_to_cpu(cqe->fast_path_cqe.
                                                                len_on_bd);
                                        bnx2x_tpa_stop(bp, fp, queue, pad,
                                                    len, cqe, comp_ring_cons);
#ifdef BNX2X_STOP_ON_ERROR
                                        if (bp->panic)
                                                return -EINVAL;
#endif

                                        bnx2x_update_sge_prod(fp,
                                                        &cqe->fast_path_cqe);
                                        goto next_cqe;
                                }
                        }

                        pci_dma_sync_single_for_device(bp->pdev,
                                        pci_unmap_addr(rx_buf, mapping),
                                                       pad + RX_COPY_THRESH,
                                                       PCI_DMA_FROMDEVICE);
                        prefetch(skb);
                        prefetch(((char *)(skb)) + 128);

                        /* is this an error packet? */
                        if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
                                DP(NETIF_MSG_RX_ERR,
                                   "ERROR  flags %x  rx packet %u\n",
                                   cqe_fp_flags, sw_comp_cons);
                                bp->eth_stats.rx_err_discard_pkt++;
                                goto reuse_rx;
                        }

                        /* Since we don't have a jumbo ring
                         * copy small packets if mtu > 1500
                         */
                        if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
                            (len <= RX_COPY_THRESH)) {
                                struct sk_buff *new_skb;

                                new_skb = netdev_alloc_skb(bp->dev,
                                                           len + pad);
                                if (new_skb == NULL) {
                                        DP(NETIF_MSG_RX_ERR,
                                           "ERROR  packet dropped "
                                           "because of alloc failure\n");
                                        bp->eth_stats.rx_skb_alloc_failed++;
                                        goto reuse_rx;
                                }

                                /* aligned copy */
                                skb_copy_from_linear_data_offset(skb, pad,
                                                    new_skb->data + pad, len);
                                skb_reserve(new_skb, pad);
                                skb_put(new_skb, len);

                                bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);

                                skb = new_skb;

                        } else if (bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0) {
                                pci_unmap_single(bp->pdev,
                                        pci_unmap_addr(rx_buf, mapping),
                                                 bp->rx_buf_size,
                                                 PCI_DMA_FROMDEVICE);
                                skb_reserve(skb, pad);
                                skb_put(skb, len);

                        } else {
                                DP(NETIF_MSG_RX_ERR,
                                   "ERROR  packet dropped because "
                                   "of alloc failure\n");
                                bp->eth_stats.rx_skb_alloc_failed++;
reuse_rx:
                                bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
                                goto next_rx;
                        }

                        skb->protocol = eth_type_trans(skb, bp->dev);

                        skb->ip_summed = CHECKSUM_NONE;
                        if (bp->rx_csum) {
                                if (likely(BNX2X_RX_CSUM_OK(cqe)))
                                        skb->ip_summed = CHECKSUM_UNNECESSARY;
                                else
                                        bp->eth_stats.hw_csum_err++;
                        }
                }

#ifdef BCM_VLAN
                if ((bp->vlgrp != NULL) &&
                    (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
                     PARSING_FLAGS_VLAN))
                        vlan_hwaccel_receive_skb(skb, bp->vlgrp,
                                le16_to_cpu(cqe->fast_path_cqe.vlan_tag));
                else
#endif
                        netif_receive_skb(skb);

                bp->dev->last_rx = jiffies;

next_rx:
                rx_buf->skb = NULL;

                bd_cons = NEXT_RX_IDX(bd_cons);
                bd_prod = NEXT_RX_IDX(bd_prod);
                bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
                rx_pkt++;
next_cqe:
                sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
                sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);

                if (rx_pkt == budget)
                        break;
        } /* while */

        fp->rx_bd_cons = bd_cons;
        fp->rx_bd_prod = bd_prod_fw;
        fp->rx_comp_cons = sw_comp_cons;
        fp->rx_comp_prod = sw_comp_prod;

        /* Update producers */
        bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
                             fp->rx_sge_prod);
        mmiowb(); /* keep prod updates ordered */

        fp->rx_pkt += rx_pkt;
        fp->rx_calls++;

        return rx_pkt;
}

static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
{
        struct bnx2x_fastpath *fp = fp_cookie;
        struct bnx2x *bp = fp->bp;
        struct net_device *dev = bp->dev;
        int index = FP_IDX(fp);

        /* Return here if interrupt is disabled */
        if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
                DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
                return IRQ_HANDLED;
        }

        DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
           index, FP_SB_ID(fp));
        bnx2x_ack_sb(bp, FP_SB_ID(fp), USTORM_ID, 0, IGU_INT_DISABLE, 0);

#ifdef BNX2X_STOP_ON_ERROR
        if (unlikely(bp->panic))
                return IRQ_HANDLED;
#endif

        prefetch(fp->rx_cons_sb);
        prefetch(fp->tx_cons_sb);
        prefetch(&fp->status_blk->c_status_block.status_block_index);
        prefetch(&fp->status_blk->u_status_block.status_block_index);

        netif_rx_schedule(dev, &bnx2x_fp(bp, index, napi));

        return IRQ_HANDLED;
}

static irqreturn_t bnx2x_interrupt(int irq, void *dev_instance)
{
        struct net_device *dev = dev_instance;
        struct bnx2x *bp = netdev_priv(dev);
        u16 status = bnx2x_ack_int(bp);
        u16 mask;

        /* Return here if interrupt is shared and it's not for us */
        if (unlikely(status == 0)) {
                DP(NETIF_MSG_INTR, "not our interrupt!\n");
                return IRQ_NONE;
        }
        DP(NETIF_MSG_INTR, "got an interrupt  status %u\n", status);

        /* Return here if interrupt is disabled */
        if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
                DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
                return IRQ_HANDLED;
        }

#ifdef BNX2X_STOP_ON_ERROR
        if (unlikely(bp->panic))
                return IRQ_HANDLED;
#endif

        mask = 0x2 << bp->fp[0].sb_id;
        if (status & mask) {
                struct bnx2x_fastpath *fp = &bp->fp[0];

                prefetch(fp->rx_cons_sb);
                prefetch(fp->tx_cons_sb);
                prefetch(&fp->status_blk->c_status_block.status_block_index);
                prefetch(&fp->status_blk->u_status_block.status_block_index);

                netif_rx_schedule(dev, &bnx2x_fp(bp, 0, napi));

                status &= ~mask;
        }


        if (unlikely(status & 0x1)) {
                schedule_work(&bp->sp_task);

                status &= ~0x1;
                if (!status)
                        return IRQ_HANDLED;
        }

        if (status)
                DP(NETIF_MSG_INTR, "got an unknown interrupt! (status %u)\n",
                   status);

        return IRQ_HANDLED;
}

/* end of fast path */

static void bnx2x_stats_handle(struct bnx2x *bp, enum bnx2x_stats_event event);

/* Link */

/*
 * General service functions
 */

static int bnx2x_acquire_hw_lock(struct bnx2x *bp, u32 resource)
{
        u32 lock_status;
        u32 resource_bit = (1 << resource);
        int func = BP_FUNC(bp);
        u32 hw_lock_control_reg;
        int cnt;

        /* Validating that the resource is within range */
        if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
                DP(NETIF_MSG_HW,
                   "resource(0x%x) > HW_LOCK_MAX_RESOURCE_VALUE(0x%x)\n",
                   resource, HW_LOCK_MAX_RESOURCE_VALUE);
                return -EINVAL;
        }

        if (func <= 5) {
                hw_lock_control_reg = (MISC_REG_DRIVER_CONTROL_1 + func*8);
        } else {
                hw_lock_control_reg =
                                (MISC_REG_DRIVER_CONTROL_7 + (func - 6)*8);
        }

        /* Validating that the resource is not already taken */
        lock_status = REG_RD(bp, hw_lock_control_reg);
        if (lock_status & resource_bit) {
                DP(NETIF_MSG_HW, "lock_status 0x%x  resource_bit 0x%x\n",
                   lock_status, resource_bit);
                return -EEXIST;
        }

        /* Try for 5 second every 5ms */
        for (cnt = 0; cnt < 1000; cnt++) {
                /* Try to acquire the lock */
                REG_WR(bp, hw_lock_control_reg + 4, resource_bit);
                lock_status = REG_RD(bp, hw_lock_control_reg);
                if (lock_status & resource_bit)
                        return 0;

                msleep(5);
        }
        DP(NETIF_MSG_HW, "Timeout\n");
        return -EAGAIN;
}

static int bnx2x_release_hw_lock(struct bnx2x *bp, u32 resource)
{
        u32 lock_status;
        u32 resource_bit = (1 << resource);
        int func = BP_FUNC(bp);
        u32 hw_lock_control_reg;

        /* Validating that the resource is within range */
        if (resource > HW_LOCK_MAX_RESOURCE_VALUE) {
                DP(NETIF_MSG_HW,
                   "resource(0x%x) > HW_LOCK_MAX_RESOURCE_VALUE(0x%x)\n",
                   resource, HW_LOCK_MAX_RESOURCE_VALUE);
                return -EINVAL;
        }

        if (func <= 5) {
                hw_lock_control_reg = (MISC_REG_DRIVER_CONTROL_1 + func*8);
        } else {
                hw_lock_control_reg =
                                (MISC_REG_DRIVER_CONTROL_7 + (func - 6)*8);
        }

        /* Validating that the resource is currently taken */
        lock_status = REG_RD(bp, hw_lock_control_reg);
        if (!(lock_status & resource_bit)) {
                DP(NETIF_MSG_HW, "lock_status 0x%x  resource_bit 0x%x\n",
                   lock_status, resource_bit);
                return -EFAULT;
        }

        REG_WR(bp, hw_lock_control_reg, resource_bit);
        return 0;
}

/* HW Lock for shared dual port PHYs */
static void bnx2x_acquire_phy_lock(struct bnx2x *bp)
{
        u32 ext_phy_type = XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);

        mutex_lock(&bp->port.phy_mutex);

        if ((ext_phy_type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8072) ||
            (ext_phy_type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8073))
                bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_8072_MDIO);
}

static void bnx2x_release_phy_lock(struct bnx2x *bp)
{
        u32 ext_phy_type = XGXS_EXT_PHY_TYPE(bp->link_params.ext_phy_config);

        if ((ext_phy_type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8072) ||
            (ext_phy_type == PORT_HW_CFG_XGXS_EXT_PHY_TYPE_BCM8073))
                bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_8072_MDIO);

        mutex_unlock(&bp->port.phy_mutex);
}

int bnx2x_set_gpio(struct bnx2x *bp, int gpio_num, u32 mode, u8 port)
{
        /* The GPIO should be swapped if swap register is set and active */
        int gpio_port = (REG_RD(bp, NIG_REG_PORT_SWAP) &&
                         REG_RD(bp, NIG_REG_STRAP_OVERRIDE)) ^ port;
        int gpio_shift = gpio_num +
                        (gpio_port ? MISC_REGISTERS_GPIO_PORT_SHIFT : 0);
        u32 gpio_mask = (1 << gpio_shift);
        u32 gpio_reg;

        if (gpio_num > MISC_REGISTERS_GPIO_3) {
                BNX2X_ERR("Invalid GPIO %d\n", gpio_num);
                return -EINVAL;
        }

        bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_GPIO);
        /* read GPIO and mask except the float bits */
        gpio_reg = (REG_RD(bp, MISC_REG_GPIO) & MISC_REGISTERS_GPIO_FLOAT);

        switch (mode) {
        case MISC_REGISTERS_GPIO_OUTPUT_LOW:
                DP(NETIF_MSG_LINK, "Set GPIO %d (shift %d) -> output low\n",
                   gpio_num, gpio_shift);
                /* clear FLOAT and set CLR */
                gpio_reg &= ~(gpio_mask << MISC_REGISTERS_GPIO_FLOAT_POS);
                gpio_reg |=  (gpio_mask << MISC_REGISTERS_GPIO_CLR_POS);
                break;

        case MISC_REGISTERS_GPIO_OUTPUT_HIGH:
                DP(NETIF_MSG_LINK, "Set GPIO %d (shift %d) -> output high\n",
                   gpio_num, gpio_shift);
                /* clear FLOAT and set SET */
                gpio_reg &= ~(gpio_mask << MISC_REGISTERS_GPIO_FLOAT_POS);
                gpio_reg |=  (gpio_mask << MISC_REGISTERS_GPIO_SET_POS);
                break;

        case MISC_REGISTERS_GPIO_INPUT_HI_Z:
                DP(NETIF_MSG_LINK, "Set GPIO %d (shift %d) -> input\n",
                   gpio_num, gpio_shift);
                /* set FLOAT */
                gpio_reg |= (gpio_mask << MISC_REGISTERS_GPIO_FLOAT_POS);
                break;

        default:
                break;
        }

        REG_WR(bp, MISC_REG_GPIO, gpio_reg);
        bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_GPIO);

        return 0;
}

static int bnx2x_set_spio(struct bnx2x *bp, int spio_num, u32 mode)
{
        u32 spio_mask = (1 << spio_num);
        u32 spio_reg;

        if ((spio_num < MISC_REGISTERS_SPIO_4) ||
            (spio_num > MISC_REGISTERS_SPIO_7)) {
                BNX2X_ERR("Invalid SPIO %d\n", spio_num);
                return -EINVAL;
        }

        bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_SPIO);
        /* read SPIO and mask except the float bits */
        spio_reg = (REG_RD(bp, MISC_REG_SPIO) & MISC_REGISTERS_SPIO_FLOAT);

        switch (mode) {
        case MISC_REGISTERS_SPIO_OUTPUT_LOW:
                DP(NETIF_MSG_LINK, "Set SPIO %d -> output low\n", spio_num);
                /* clear FLOAT and set CLR */
                spio_reg &= ~(spio_mask << MISC_REGISTERS_SPIO_FLOAT_POS);
                spio_reg |=  (spio_mask << MISC_REGISTERS_SPIO_CLR_POS);
                break;

        case MISC_REGISTERS_SPIO_OUTPUT_HIGH:
                DP(NETIF_MSG_LINK, "Set SPIO %d -> output high\n", spio_num);
                /* clear FLOAT and set SET */
                spio_reg &= ~(spio_mask << MISC_REGISTERS_SPIO_FLOAT_POS);
                spio_reg |=  (spio_mask << MISC_REGISTERS_SPIO_SET_POS);
                break;

        case MISC_REGISTERS_SPIO_INPUT_HI_Z:
                DP(NETIF_MSG_LINK, "Set SPIO %d -> input\n", spio_num);
                /* set FLOAT */
                spio_reg |= (spio_mask << MISC_REGISTERS_SPIO_FLOAT_POS);
                break;

        default:
                break;
        }

        REG_WR(bp, MISC_REG_SPIO, spio_reg);
        bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_SPIO);

        return 0;
}

static void bnx2x_calc_fc_adv(struct bnx2x *bp)
{
        switch (bp->link_vars.ieee_fc) {
        case MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_NONE:
                bp->port.advertising &= ~(ADVERTISED_Asym_Pause |
                                          ADVERTISED_Pause);
                break;
        case MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_BOTH:
                bp->port.advertising |= (ADVERTISED_Asym_Pause |
                                         ADVERTISED_Pause);
                break;
        case MDIO_COMBO_IEEE0_AUTO_NEG_ADV_PAUSE_ASYMMETRIC:
                bp->port.advertising |= ADVERTISED_Asym_Pause;
                break;
        default:
                bp->port.advertising &= ~(ADVERTISED_Asym_Pause |
                                          ADVERTISED_Pause);
                break;
        }
}

static void bnx2x_link_report(struct bnx2x *bp)
{
        if (bp->link_vars.link_up) {
                if (bp->state == BNX2X_STATE_OPEN)
                        netif_carrier_on(bp->dev);
                printk(KERN_INFO PFX "%s NIC Link is Up, ", bp->dev->name);

                printk("%d Mbps ", bp->link_vars.line_speed);

                if (bp->link_vars.duplex == DUPLEX_FULL)
                        printk("full duplex");
                else
                        printk("half duplex");

                if (bp->link_vars.flow_ctrl != FLOW_CTRL_NONE) {
                        if (bp->link_vars.flow_ctrl & FLOW_CTRL_RX) {
                                printk(", receive ");
                                if (bp->link_vars.flow_ctrl & FLOW_CTRL_TX)
                                        printk("& transmit ");
                        } else {
                                printk(", transmit ");
                        }
                        printk("flow control ON");
                }
                printk("\n");

        } else { /* link_down */
                netif_carrier_off(bp->dev);
                printk(KERN_ERR PFX "%s NIC Link is Down\n", bp->dev->name);
        }
}

static u8 bnx2x_initial_phy_init(struct bnx2x *bp)
{
        if (!BP_NOMCP(bp)) {
                u8 rc;

                /* Initialize link parameters structure variables */
                /* It is recommended to turn off RX FC for jumbo frames
                   for better performance */
                if (IS_E1HMF(bp))
                        bp->link_params.req_fc_auto_adv = FLOW_CTRL_BOTH;
                else if (bp->dev->mtu > 5000)
                        bp->link_params.req_fc_auto_adv = FLOW_CTRL_TX;
                else
                        bp->link_params.req_fc_auto_adv = FLOW_CTRL_BOTH;

                bnx2x_acquire_phy_lock(bp);
                rc = bnx2x_phy_init(&bp->link_params, &bp->link_vars);
                bnx2x_release_phy_lock(bp);

                if (bp->link_vars.link_up)
                        bnx2x_link_report(bp);

                bnx2x_calc_fc_adv(bp);

                return rc;
        }
        BNX2X_ERR("Bootcode is missing -not initializing link\n");
        return -EINVAL;
}

static void bnx2x_link_set(struct bnx2x *bp)
{
        if (!BP_NOMCP(bp)) {
                bnx2x_acquire_phy_lock(bp);
                bnx2x_phy_init(&bp->link_params, &bp->link_vars);
                bnx2x_release_phy_lock(bp);

                bnx2x_calc_fc_adv(bp);
        } else
                BNX2X_ERR("Bootcode is missing -not setting link\n");
}

static void bnx2x__link_reset(struct bnx2x *bp)
{
        if (!BP_NOMCP(bp)) {
                bnx2x_acquire_phy_lock(bp);
                bnx2x_link_reset(&bp->link_params, &bp->link_vars);
                bnx2x_release_phy_lock(bp);
        } else
                BNX2X_ERR("Bootcode is missing -not resetting link\n");
}

static u8 bnx2x_link_test(struct bnx2x *bp)
{
        u8 rc;

        bnx2x_acquire_phy_lock(bp);

        rc = bnx2x_test_link(&bp->link_params, &bp->link_vars);
        bnx2x_release_phy_lock(bp);

        return rc;
}

/* Calculates the sum of vn_min_rates.
   It's needed for further normalizing of the min_rates.

   Returns:
     sum of vn_min_rates
       or
     0 - if all the min_rates are 0.
     In the later case fairness algorithm should be deactivated.
     If not all min_rates are zero then those that are zeroes will
     be set to 1.
 */
static u32 bnx2x_calc_vn_wsum(struct bnx2x *bp)
{
        int i, port = BP_PORT(bp);
        u32 wsum = 0;
        int all_zero = 1;

        for (i = 0; i < E1HVN_MAX; i++) {
                u32 vn_cfg =
                        SHMEM_RD(bp, mf_cfg.func_mf_config[2*i + port].config);
                u32 vn_min_rate = ((vn_cfg & FUNC_MF_CFG_MIN_BW_MASK) >>
                                     FUNC_MF_CFG_MIN_BW_SHIFT) * 100;
                if (!(vn_cfg & FUNC_MF_CFG_FUNC_HIDE)) {
                        /* If min rate is zero - set it to 1 */
                        if (!vn_min_rate)
                                vn_min_rate = DEF_MIN_RATE;
                        else
                                all_zero = 0;

                        wsum += vn_min_rate;
                }
        }

        /* ... only if all min rates are zeros - disable FAIRNESS */
        if (all_zero)
                return 0;

        return wsum;
}

static void bnx2x_init_port_minmax(struct bnx2x *bp,
                                   int en_fness,
                                   u16 port_rate,
                                   struct cmng_struct_per_port *m_cmng_port)
{
        u32 r_param = port_rate / 8;
        int port = BP_PORT(bp);
        int i;

        memset(m_cmng_port, 0, sizeof(struct cmng_struct_per_port));

        /* Enable minmax only if we are in e1hmf mode */
        if (IS_E1HMF(bp)) {
                u32 fair_periodic_timeout_usec;
                u32 t_fair;

                /* Enable rate shaping and fairness */
                m_cmng_port->flags.cmng_vn_enable = 1;
                m_cmng_port->flags.fairness_enable = en_fness ? 1 : 0;
                m_cmng_port->flags.rate_shaping_enable = 1;

                if (!en_fness)
                        DP(NETIF_MSG_IFUP, "All MIN values are zeroes"
                           "  fairness will be disabled\n");

                /* 100 usec in SDM ticks = 25 since each tick is 4 usec */
                m_cmng_port->rs_vars.rs_periodic_timeout =
                                                RS_PERIODIC_TIMEOUT_USEC / 4;

                /* this is the threshold below which no timer arming will occur
                   1.25 coefficient is for the threshold to be a little bigger
                   than the real time, to compensate for timer in-accuracy */
                m_cmng_port->rs_vars.rs_threshold =
                                (RS_PERIODIC_TIMEOUT_USEC * r_param * 5) / 4;

                /* resolution of fairness timer */
                fair_periodic_timeout_usec = QM_ARB_BYTES / r_param;
                /* for 10G it is 1000usec. for 1G it is 10000usec. */
                t_fair = T_FAIR_COEF / port_rate;

                /* this is the threshold below which we won't arm
                   the timer anymore */
                m_cmng_port->fair_vars.fair_threshold = QM_ARB_BYTES;

                /* we multiply by 1e3/8 to get bytes/msec.
                   We don't want the credits to pass a credit
                   of the T_FAIR*FAIR_MEM (algorithm resolution) */
                m_cmng_port->fair_vars.upper_bound =
                                                r_param * t_fair * FAIR_MEM;
                /* since each tick is 4 usec */
                m_cmng_port->fair_vars.fairness_timeout =
                                                fair_periodic_timeout_usec / 4;

        } else {
                /* Disable rate shaping and fairness */
                m_cmng_port->flags.cmng_vn_enable = 0;
                m_cmng_port->flags.fairness_enable = 0;
                m_cmng_port->flags.rate_shaping_enable = 0;

                DP(NETIF_MSG_IFUP,
                   "Single function mode  minmax will be disabled\n");
        }

        /* Store it to internal memory */
        for (i = 0; i < sizeof(struct cmng_struct_per_port) / 4; i++)
                REG_WR(bp, BAR_XSTRORM_INTMEM +
                       XSTORM_CMNG_PER_PORT_VARS_OFFSET(port) + i * 4,
                       ((u32 *)(m_cmng_port))[i]);
}

static void bnx2x_init_vn_minmax(struct bnx2x *bp, int func,
                                   u32 wsum, u16 port_rate,
                                 struct cmng_struct_per_port *m_cmng_port)
{
        struct rate_shaping_vars_per_vn m_rs_vn;
        struct fairness_vars_per_vn m_fair_vn;
        u32 vn_cfg = SHMEM_RD(bp, mf_cfg.func_mf_config[func].config);
        u16 vn_min_rate, vn_max_rate;
        int i;

        /* If function is hidden - set min and max to zeroes */
        if (vn_cfg & FUNC_MF_CFG_FUNC_HIDE) {
                vn_min_rate = 0;
                vn_max_rate = 0;

        } else {
                vn_min_rate = ((vn_cfg & FUNC_MF_CFG_MIN_BW_MASK) >>
                                FUNC_MF_CFG_MIN_BW_SHIFT) * 100;
                /* If FAIRNESS is enabled (not all min rates are zeroes) and
                   if current min rate is zero - set it to 1.
                   This is a requirement of the algorithm. */
                if ((vn_min_rate == 0) && wsum)
                        vn_min_rate = DEF_MIN_RATE;
                vn_max_rate = ((vn_cfg & FUNC_MF_CFG_MAX_BW_MASK) >>
                                FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
        }

        DP(NETIF_MSG_IFUP, "func %d: vn_min_rate=%d  vn_max_rate=%d  "
           "wsum=%d\n", func, vn_min_rate, vn_max_rate, wsum);

        memset(&m_rs_vn, 0, sizeof(struct rate_shaping_vars_per_vn));
        memset(&m_fair_vn, 0, sizeof(struct fairness_vars_per_vn));

        /* global vn counter - maximal Mbps for this vn */
        m_rs_vn.vn_counter.rate = vn_max_rate;

        /* quota - number of bytes transmitted in this period */
        m_rs_vn.vn_counter.quota =
                                (vn_max_rate * RS_PERIODIC_TIMEOUT_USEC) / 8;

#ifdef BNX2X_PER_PROT_QOS
        /* per protocol counter */
        for (protocol = 0; protocol < NUM_OF_PROTOCOLS; protocol++) {
                /* maximal Mbps for this protocol */
                m_rs_vn.protocol_counters[protocol].rate =
                                                protocol_max_rate[protocol];
                /* the quota in each timer period -
                   number of bytes transmitted in this period */
                m_rs_vn.protocol_counters[protocol].quota =
                        (u32)(rs_periodic_timeout_usec *
                          ((double)m_rs_vn.
                                   protocol_counters[protocol].rate/8));
        }
#endif

        if (wsum) {
                /* credit for each period of the fairness algorithm:
                   number of bytes in T_FAIR (the vn share the port rate).
                   wsum should not be larger than 10000, thus
                   T_FAIR_COEF / (8 * wsum) will always be grater than zero */
                m_fair_vn.vn_credit_delta =
                        max((u64)(vn_min_rate * (T_FAIR_COEF / (8 * wsum))),
                            (u64)(m_cmng_port->fair_vars.fair_threshold * 2));
                DP(NETIF_MSG_IFUP, "m_fair_vn.vn_credit_delta=%d\n",
                   m_fair_vn.vn_credit_delta);
        }

#ifdef BNX2X_PER_PROT_QOS
        do {
                u32 protocolWeightSum = 0;

                for (protocol = 0; protocol < NUM_OF_PROTOCOLS; protocol++)
                        protocolWeightSum +=
                                        drvInit.protocol_min_rate[protocol];
                /* per protocol counter -
                   NOT NEEDED IF NO PER-PROTOCOL CONGESTION MANAGEMENT */
                if (protocolWeightSum > 0) {
                        for (protocol = 0;
                             protocol < NUM_OF_PROTOCOLS; protocol++)
                                /* credit for each period of the
                                   fairness algorithm - number of bytes in
                                   T_FAIR (the protocol share the vn rate) */
                                m_fair_vn.protocol_credit_delta[protocol] =
                                        (u32)((vn_min_rate / 8) * t_fair *
                                        protocol_min_rate / protocolWeightSum);
                }
        } while (0);
#endif

        /* Store it to internal memory */
        for (i = 0; i < sizeof(struct rate_shaping_vars_per_vn)/4; i++)
                REG_WR(bp, BAR_XSTRORM_INTMEM +
                       XSTORM_RATE_SHAPING_PER_VN_VARS_OFFSET(func) + i * 4,
                       ((u32 *)(&m_rs_vn))[i]);

        for (i = 0; i < sizeof(struct fairness_vars_per_vn)/4; i++)
                REG_WR(bp, BAR_XSTRORM_INTMEM +
                       XSTORM_FAIRNESS_PER_VN_VARS_OFFSET(func) + i * 4,
                       ((u32 *)(&m_fair_vn))[i]);
}

/* This function is called upon link interrupt */
static void bnx2x_link_attn(struct bnx2x *bp)
{
        int vn;

        /* Make sure that we are synced with the current statistics */
        bnx2x_stats_handle(bp, STATS_EVENT_STOP);

        bnx2x_acquire_phy_lock(bp);
        bnx2x_link_update(&bp->link_params, &bp->link_vars);
        bnx2x_release_phy_lock(bp);

        if (bp->link_vars.link_up) {

                if (bp->link_vars.mac_type == MAC_TYPE_BMAC) {
                        struct host_port_stats *pstats;

                        pstats = bnx2x_sp(bp, port_stats);
                        /* reset old bmac stats */
                        memset(&(pstats->mac_stx[0]), 0,
                               sizeof(struct mac_stx));
                }
                if ((bp->state == BNX2X_STATE_OPEN) ||
                    (bp->state == BNX2X_STATE_DISABLED))
                        bnx2x_stats_handle(bp, STATS_EVENT_LINK_UP);
        }

        /* indicate link status */
        bnx2x_link_report(bp);

        if (IS_E1HMF(bp)) {
                int func;

                for (vn = VN_0; vn < E1HVN_MAX; vn++) {
                        if (vn == BP_E1HVN(bp))
                                continue;

                        func = ((vn << 1) | BP_PORT(bp));

                        /* Set the attention towards other drivers
                           on the same port */
                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_0 +
                               (LINK_SYNC_ATTENTION_BIT_FUNC_0 + func)*4, 1);
                }
        }

        if (CHIP_IS_E1H(bp) && (bp->link_vars.line_speed > 0)) {
                struct cmng_struct_per_port m_cmng_port;
                u32 wsum;
                int port = BP_PORT(bp);

                /* Init RATE SHAPING and FAIRNESS contexts */
                wsum = bnx2x_calc_vn_wsum(bp);
                bnx2x_init_port_minmax(bp, (int)wsum,
                                        bp->link_vars.line_speed,
                                        &m_cmng_port);
                if (IS_E1HMF(bp))
                        for (vn = VN_0; vn < E1HVN_MAX; vn++)
                                bnx2x_init_vn_minmax(bp, 2*vn + port,
                                        wsum, bp->link_vars.line_speed,
                                                     &m_cmng_port);
        }
}

static void bnx2x__link_status_update(struct bnx2x *bp)
{
        if (bp->state != BNX2X_STATE_OPEN)
                return;

        bnx2x_link_status_update(&bp->link_params, &bp->link_vars);

        if (bp->link_vars.link_up)
                bnx2x_stats_handle(bp, STATS_EVENT_LINK_UP);
        else
                bnx2x_stats_handle(bp, STATS_EVENT_STOP);

        /* indicate link status */
        bnx2x_link_report(bp);
}

static void bnx2x_pmf_update(struct bnx2x *bp)
{
        int port = BP_PORT(bp);
        u32 val;

        bp->port.pmf = 1;
        DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);

        /* enable nig attention */
        val = (0xff0f | (1 << (BP_E1HVN(bp) + 4)));
        REG_WR(bp, HC_REG_TRAILING_EDGE_0 + port*8, val);
        REG_WR(bp, HC_REG_LEADING_EDGE_0 + port*8, val);

        bnx2x_stats_handle(bp, STATS_EVENT_PMF);
}

/* end of Link */

/* slow path */

/*
 * General service functions
 */

/* the slow path queue is odd since completions arrive on the fastpath ring */
static int bnx2x_sp_post(struct bnx2x *bp, int command, int cid,
                         u32 data_hi, u32 data_lo, int common)
{
        int func = BP_FUNC(bp);

        DP(BNX2X_MSG_SP/*NETIF_MSG_TIMER*/,
           "SPQE (%x:%x)  command %d  hw_cid %x  data (%x:%x)  left %x\n",
           (u32)U64_HI(bp->spq_mapping), (u32)(U64_LO(bp->spq_mapping) +
           (void *)bp->spq_prod_bd - (void *)bp->spq), command,
           HW_CID(bp, cid), data_hi, data_lo, bp->spq_left);

#ifdef BNX2X_STOP_ON_ERROR
        if (unlikely(bp->panic))
                return -EIO;
#endif

        spin_lock_bh(&bp->spq_lock);

        if (!bp->spq_left) {
                BNX2X_ERR("BUG! SPQ ring full!\n");
                spin_unlock_bh(&bp->spq_lock);
                bnx2x_panic();
                return -EBUSY;
        }

        /* CID needs port number to be encoded int it */
        bp->spq_prod_bd->hdr.conn_and_cmd_data =
                        cpu_to_le32(((command << SPE_HDR_CMD_ID_SHIFT) |
                                     HW_CID(bp, cid)));
        bp->spq_prod_bd->hdr.type = cpu_to_le16(ETH_CONNECTION_TYPE);
        if (common)
                bp->spq_prod_bd->hdr.type |=
                        cpu_to_le16((1 << SPE_HDR_COMMON_RAMROD_SHIFT));

        bp->spq_prod_bd->data.mac_config_addr.hi = cpu_to_le32(data_hi);
        bp->spq_prod_bd->data.mac_config_addr.lo = cpu_to_le32(data_lo);

        bp->spq_left--;

        if (bp->spq_prod_bd == bp->spq_last_bd) {
                bp->spq_prod_bd = bp->spq;
                bp->spq_prod_idx = 0;
                DP(NETIF_MSG_TIMER, "end of spq\n");

        } else {
                bp->spq_prod_bd++;
                bp->spq_prod_idx++;
        }

        REG_WR(bp, BAR_XSTRORM_INTMEM + XSTORM_SPQ_PROD_OFFSET(func),
               bp->spq_prod_idx);

        spin_unlock_bh(&bp->spq_lock);
        return 0;
}

/* acquire split MCP access lock register */
static int bnx2x_acquire_alr(struct bnx2x *bp)
{
        u32 i, j, val;
        int rc = 0;

        might_sleep();
        i = 100;
        for (j = 0; j < i*10; j++) {
                val = (1UL << 31);
                REG_WR(bp, GRCBASE_MCP + 0x9c, val);
                val = REG_RD(bp, GRCBASE_MCP + 0x9c);
                if (val & (1L << 31))
                        break;

                msleep(5);
        }
        if (!(val & (1L << 31))) {
                BNX2X_ERR("Cannot acquire MCP access lock register\n");
                rc = -EBUSY;
        }

        return rc;
}

/* release split MCP access lock register */
static void bnx2x_release_alr(struct bnx2x *bp)
{
        u32 val = 0;

        REG_WR(bp, GRCBASE_MCP + 0x9c, val);
}

static inline u16 bnx2x_update_dsb_idx(struct bnx2x *bp)
{
        struct host_def_status_block *def_sb = bp->def_status_blk;
        u16 rc = 0;

        barrier(); /* status block is written to by the chip */
        if (bp->def_att_idx != def_sb->atten_status_block.attn_bits_index) {
                bp->def_att_idx = def_sb->atten_status_block.attn_bits_index;
                rc |= 1;
        }
        if (bp->def_c_idx != def_sb->c_def_status_block.status_block_index) {
                bp->def_c_idx = def_sb->c_def_status_block.status_block_index;
                rc |= 2;
        }
        if (bp->def_u_idx != def_sb->u_def_status_block.status_block_index) {
                bp->def_u_idx = def_sb->u_def_status_block.status_block_index;
                rc |= 4;
        }
        if (bp->def_x_idx != def_sb->x_def_status_block.status_block_index) {
                bp->def_x_idx = def_sb->x_def_status_block.status_block_index;
                rc |= 8;
        }
        if (bp->def_t_idx != def_sb->t_def_status_block.status_block_index) {
                bp->def_t_idx = def_sb->t_def_status_block.status_block_index;
                rc |= 16;
        }
        return rc;
}

/*
 * slow path service functions
 */

static void bnx2x_attn_int_asserted(struct bnx2x *bp, u32 asserted)
{
        int port = BP_PORT(bp);
        u32 hc_addr = (HC_REG_COMMAND_REG + port*32 +
                       COMMAND_REG_ATTN_BITS_SET);
        u32 aeu_addr = port ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
                              MISC_REG_AEU_MASK_ATTN_FUNC_0;
        u32 nig_int_mask_addr = port ? NIG_REG_MASK_INTERRUPT_PORT1 :
                                       NIG_REG_MASK_INTERRUPT_PORT0;
        u32 aeu_mask;

        if (bp->attn_state & asserted)
                BNX2X_ERR("IGU ERROR\n");

        bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
        aeu_mask = REG_RD(bp, aeu_addr);

        DP(NETIF_MSG_HW, "aeu_mask %x  newly asserted %x\n",
           aeu_mask, asserted);
        aeu_mask &= ~(asserted & 0xff);
        DP(NETIF_MSG_HW, "new mask %x\n", aeu_mask);

        REG_WR(bp, aeu_addr, aeu_mask);
        bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);

        DP(NETIF_MSG_HW, "attn_state %x\n", bp->attn_state);
        bp->attn_state |= asserted;
        DP(NETIF_MSG_HW, "new state %x\n", bp->attn_state);

        if (asserted & ATTN_HARD_WIRED_MASK) {
                if (asserted & ATTN_NIG_FOR_FUNC) {

                        /* save nig interrupt mask */
                        bp->nig_mask = REG_RD(bp, nig_int_mask_addr);
                        REG_WR(bp, nig_int_mask_addr, 0);

                        bnx2x_link_attn(bp);

                        /* handle unicore attn? */
                }
                if (asserted & ATTN_SW_TIMER_4_FUNC)
                        DP(NETIF_MSG_HW, "ATTN_SW_TIMER_4_FUNC!\n");

                if (asserted & GPIO_2_FUNC)
                        DP(NETIF_MSG_HW, "GPIO_2_FUNC!\n");

                if (asserted & GPIO_3_FUNC)
                        DP(NETIF_MSG_HW, "GPIO_3_FUNC!\n");

                if (asserted & GPIO_4_FUNC)
                        DP(NETIF_MSG_HW, "GPIO_4_FUNC!\n");

                if (port == 0) {
                        if (asserted & ATTN_GENERAL_ATTN_1) {
                                DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_1!\n");
                                REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_1, 0x0);
                        }
                        if (asserted & ATTN_GENERAL_ATTN_2) {
                                DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_2!\n");
                                REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_2, 0x0);
                        }
                        if (asserted & ATTN_GENERAL_ATTN_3) {
                                DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_3!\n");
                                REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_3, 0x0);
                        }
                } else {
                        if (asserted & ATTN_GENERAL_ATTN_4) {
                                DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_4!\n");
                                REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_4, 0x0);
                        }
                        if (asserted & ATTN_GENERAL_ATTN_5) {
                                DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_5!\n");
                                REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_5, 0x0);
                        }
                        if (asserted & ATTN_GENERAL_ATTN_6) {
                                DP(NETIF_MSG_HW, "ATTN_GENERAL_ATTN_6!\n");
                                REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_6, 0x0);
                        }
                }

        } /* if hardwired */

        DP(NETIF_MSG_HW, "about to mask 0x%08x at HC addr 0x%x\n",
           asserted, hc_addr);
        REG_WR(bp, hc_addr, asserted);

        /* now set back the mask */
        if (asserted & ATTN_NIG_FOR_FUNC)
                REG_WR(bp, nig_int_mask_addr, bp->nig_mask);
}

static inline void bnx2x_attn_int_deasserted0(struct bnx2x *bp, u32 attn)
{
        int port = BP_PORT(bp);
        int reg_offset;
        u32 val;

        reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_0 :
                             MISC_REG_AEU_ENABLE1_FUNC_0_OUT_0);

        if (attn & AEU_INPUTS_ATTN_BITS_SPIO5) {

                val = REG_RD(bp, reg_offset);
                val &= ~AEU_INPUTS_ATTN_BITS_SPIO5;
                REG_WR(bp, reg_offset, val);

                BNX2X_ERR("SPIO5 hw attention\n");

                switch (bp->common.board & SHARED_HW_CFG_BOARD_TYPE_MASK) {
                case SHARED_HW_CFG_BOARD_TYPE_BCM957710A1021G:
                case SHARED_HW_CFG_BOARD_TYPE_BCM957710A1022G:
                        /* Fan failure attention */

                        /* The PHY reset is controlled by GPIO 1 */
                        bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_1,
                                       MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
                        /* Low power mode is controlled by GPIO 2 */
                        bnx2x_set_gpio(bp, MISC_REGISTERS_GPIO_2,
                                       MISC_REGISTERS_GPIO_OUTPUT_LOW, port);
                        /* mark the failure */
                        bp->link_params.ext_phy_config &=
                                        ~PORT_HW_CFG_XGXS_EXT_PHY_TYPE_MASK;
                        bp->link_params.ext_phy_config |=
                                        PORT_HW_CFG_XGXS_EXT_PHY_TYPE_FAILURE;
                        SHMEM_WR(bp,
                                 dev_info.port_hw_config[port].
                                                        external_phy_config,
                                 bp->link_params.ext_phy_config);
                        /* log the failure */
                        printk(KERN_ERR PFX "Fan Failure on Network"
                               " Controller %s has caused the driver to"
                               " shutdown the card to prevent permanent"
                               " damage.  Please contact Dell Support for"
                               " assistance\n", bp->dev->name);
                        break;

                default:
                        break;
                }
        }

        if (attn & HW_INTERRUT_ASSERT_SET_0) {

                val = REG_RD(bp, reg_offset);
                val &= ~(attn & HW_INTERRUT_ASSERT_SET_0);
                REG_WR(bp, reg_offset, val);

                BNX2X_ERR("FATAL HW block attention set0 0x%x\n",
                          (attn & HW_INTERRUT_ASSERT_SET_0));
                bnx2x_panic();
        }
}

static inline void bnx2x_attn_int_deasserted1(struct bnx2x *bp, u32 attn)
{
        u32 val;

        if (attn & BNX2X_DOORQ_ASSERT) {

                val = REG_RD(bp, DORQ_REG_DORQ_INT_STS_CLR);
                BNX2X_ERR("DB hw attention 0x%x\n", val);
                /* DORQ discard attention */
                if (val & 0x2)
                        BNX2X_ERR("FATAL error from DORQ\n");
        }

        if (attn & HW_INTERRUT_ASSERT_SET_1) {

                int port = BP_PORT(bp);
                int reg_offset;

                reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_1 :
                                     MISC_REG_AEU_ENABLE1_FUNC_0_OUT_1);

                val = REG_RD(bp, reg_offset);
                val &= ~(attn & HW_INTERRUT_ASSERT_SET_1);
                REG_WR(bp, reg_offset, val);

                BNX2X_ERR("FATAL HW block attention set1 0x%x\n",
                          (attn & HW_INTERRUT_ASSERT_SET_1));
                bnx2x_panic();
        }
}

static inline void bnx2x_attn_int_deasserted2(struct bnx2x *bp, u32 attn)
{
        u32 val;

        if (attn & AEU_INPUTS_ATTN_BITS_CFC_HW_INTERRUPT) {

                val = REG_RD(bp, CFC_REG_CFC_INT_STS_CLR);
                BNX2X_ERR("CFC hw attention 0x%x\n", val);
                /* CFC error attention */
                if (val & 0x2)
                        BNX2X_ERR("FATAL error from CFC\n");
        }

        if (attn & AEU_INPUTS_ATTN_BITS_PXP_HW_INTERRUPT) {

                val = REG_RD(bp, PXP_REG_PXP_INT_STS_CLR_0);
                BNX2X_ERR("PXP hw attention 0x%x\n", val);
                /* RQ_USDMDP_FIFO_OVERFLOW */
                if (val & 0x18000)
                        BNX2X_ERR("FATAL error from PXP\n");
        }

        if (attn & HW_INTERRUT_ASSERT_SET_2) {

                int port = BP_PORT(bp);
                int reg_offset;

                reg_offset = (port ? MISC_REG_AEU_ENABLE1_FUNC_1_OUT_2 :
                                     MISC_REG_AEU_ENABLE1_FUNC_0_OUT_2);

                val = REG_RD(bp, reg_offset);
                val &= ~(attn & HW_INTERRUT_ASSERT_SET_2);
                REG_WR(bp, reg_offset, val);

                BNX2X_ERR("FATAL HW block attention set2 0x%x\n",
                          (attn & HW_INTERRUT_ASSERT_SET_2));
                bnx2x_panic();
        }
}

static inline void bnx2x_attn_int_deasserted3(struct bnx2x *bp, u32 attn)
{
        u32 val;

        if (attn & EVEREST_GEN_ATTN_IN_USE_MASK) {

                if (attn & BNX2X_PMF_LINK_ASSERT) {
                        int func = BP_FUNC(bp);

                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_12 + func*4, 0);
                        bnx2x__link_status_update(bp);
                        if (SHMEM_RD(bp, func_mb[func].drv_status) &
                                                        DRV_STATUS_PMF)
                                bnx2x_pmf_update(bp);

                } else if (attn & BNX2X_MC_ASSERT_BITS) {

                        BNX2X_ERR("MC assert!\n");
                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_10, 0);
                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_9, 0);
                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_8, 0);
                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_7, 0);
                        bnx2x_panic();

                } else if (attn & BNX2X_MCP_ASSERT) {

                        BNX2X_ERR("MCP assert!\n");
                        REG_WR(bp, MISC_REG_AEU_GENERAL_ATTN_11, 0);
                        bnx2x_fw_dump(bp);

                } else
                        BNX2X_ERR("Unknown HW assert! (attn 0x%x)\n", attn);
        }

        if (attn & EVEREST_LATCHED_ATTN_IN_USE_MASK) {
                BNX2X_ERR("LATCHED attention 0x%08x (masked)\n", attn);
                if (attn & BNX2X_GRC_TIMEOUT) {
                        val = CHIP_IS_E1H(bp) ?
                                REG_RD(bp, MISC_REG_GRC_TIMEOUT_ATTN) : 0;
                        BNX2X_ERR("GRC time-out 0x%08x\n", val);
                }
                if (attn & BNX2X_GRC_RSV) {
                        val = CHIP_IS_E1H(bp) ?
                                REG_RD(bp, MISC_REG_GRC_RSV_ATTN) : 0;
                        BNX2X_ERR("GRC reserved 0x%08x\n", val);
                }
                REG_WR(bp, MISC_REG_AEU_CLR_LATCH_SIGNAL, 0x7ff);
        }
}

static void bnx2x_attn_int_deasserted(struct bnx2x *bp, u32 deasserted)
{
        struct attn_route attn;
        struct attn_route group_mask;
        int port = BP_PORT(bp);
        int index;
        u32 reg_addr;
        u32 val;
        u32 aeu_mask;

        /* need to take HW lock because MCP or other port might also
           try to handle this event */
        bnx2x_acquire_alr(bp);

        attn.sig[0] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_1_FUNC_0 + port*4);
        attn.sig[1] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_2_FUNC_0 + port*4);
        attn.sig[2] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_3_FUNC_0 + port*4);
        attn.sig[3] = REG_RD(bp, MISC_REG_AEU_AFTER_INVERT_4_FUNC_0 + port*4);
        DP(NETIF_MSG_HW, "attn: %08x %08x %08x %08x\n",
           attn.sig[0], attn.sig[1], attn.sig[2], attn.sig[3]);

        for (index = 0; index < MAX_DYNAMIC_ATTN_GRPS; index++) {
                if (deasserted & (1 << index)) {
                        group_mask = bp->attn_group[index];

                        DP(NETIF_MSG_HW, "group[%d]: %08x %08x %08x %08x\n",
                           index, group_mask.sig[0], group_mask.sig[1],
                           group_mask.sig[2], group_mask.sig[3]);

                        bnx2x_attn_int_deasserted3(bp,
                                        attn.sig[3] & group_mask.sig[3]);
                        bnx2x_attn_int_deasserted1(bp,
                                        attn.sig[1] & group_mask.sig[1]);
                        bnx2x_attn_int_deasserted2(bp,
                                        attn.sig[2] & group_mask.sig[2]);
                        bnx2x_attn_int_deasserted0(bp,
                                        attn.sig[0] & group_mask.sig[0]);

                        if ((attn.sig[0] & group_mask.sig[0] &
                                                HW_PRTY_ASSERT_SET_0) ||
                            (attn.sig[1] & group_mask.sig[1] &
                                                HW_PRTY_ASSERT_SET_1) ||
                            (attn.sig[2] & group_mask.sig[2] &
                                                HW_PRTY_ASSERT_SET_2))
                                BNX2X_ERR("FATAL HW block parity attention\n");
                }
        }

        bnx2x_release_alr(bp);

        reg_addr = (HC_REG_COMMAND_REG + port*32 + COMMAND_REG_ATTN_BITS_CLR);

        val = ~deasserted;
        DP(NETIF_MSG_HW, "about to mask 0x%08x at HC addr 0x%x\n",
           val, reg_addr);
        REG_WR(bp, reg_addr, val);

        if (~bp->attn_state & deasserted)
                BNX2X_ERR("IGU ERROR\n");

        reg_addr = port ? MISC_REG_AEU_MASK_ATTN_FUNC_1 :
                          MISC_REG_AEU_MASK_ATTN_FUNC_0;

        bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);
        aeu_mask = REG_RD(bp, reg_addr);

        DP(NETIF_MSG_HW, "aeu_mask %x  newly deasserted %x\n",
           aeu_mask, deasserted);
        aeu_mask |= (deasserted & 0xff);
        DP(NETIF_MSG_HW, "new mask %x\n", aeu_mask);

        REG_WR(bp, reg_addr, aeu_mask);
        bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_PORT0_ATT_MASK + port);

        DP(NETIF_MSG_HW, "attn_state %x\n", bp->attn_state);
        bp->attn_state &= ~deasserted;
        DP(NETIF_MSG_HW, "new state %x\n", bp->attn_state);
}

static void bnx2x_attn_int(struct bnx2x *bp)
{
        /* read local copy of bits */
        u32 attn_bits = bp->def_status_blk->atten_status_block.attn_bits;
        u32 attn_ack = bp->def_status_blk->atten_status_block.attn_bits_ack;
        u32 attn_state = bp->attn_state;

        /* look for changed bits */
        u32 asserted   =  attn_bits & ~attn_ack & ~attn_state;
        u32 deasserted = ~attn_bits &  attn_ack &  attn_state;

        DP(NETIF_MSG_HW,
           "attn_bits %x  attn_ack %x  asserted %x  deasserted %x\n",
           attn_bits, attn_ack, asserted, deasserted);

        if (~(attn_bits ^ attn_ack) & (attn_bits ^ attn_state))
                BNX2X_ERR("BAD attention state\n");

        /* handle bits that were raised */
        if (asserted)
                bnx2x_attn_int_asserted(bp, asserted);

        if (deasserted)
                bnx2x_attn_int_deasserted(bp, deasserted);
}

static void bnx2x_sp_task(struct work_struct *work)
{
        struct bnx2x *bp = container_of(work, struct bnx2x, sp_task);
        u16 status;


        /* Return here if interrupt is disabled */
        if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
                DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
                return;
        }

        status = bnx2x_update_dsb_idx(bp);
/*      if (status == 0)                                     */
/*              BNX2X_ERR("spurious slowpath interrupt!\n"); */

        DP(NETIF_MSG_INTR, "got a slowpath interrupt (updated %x)\n", status);

        /* HW attentions */
        if (status & 0x1)
                bnx2x_attn_int(bp);

        /* CStorm events: query_stats, port delete ramrod */
        if (status & 0x2)
                bp->stats_pending = 0;

        bnx2x_ack_sb(bp, DEF_SB_ID, ATTENTION_ID, bp->def_att_idx,
                     IGU_INT_NOP, 1);
        bnx2x_ack_sb(bp, DEF_SB_ID, USTORM_ID, le16_to_cpu(bp->def_u_idx),
                     IGU_INT_NOP, 1);
        bnx2x_ack_sb(bp, DEF_SB_ID, CSTORM_ID, le16_to_cpu(bp->def_c_idx),
                     IGU_INT_NOP, 1);
        bnx2x_ack_sb(bp, DEF_SB_ID, XSTORM_ID, le16_to_cpu(bp->def_x_idx),
                     IGU_INT_NOP, 1);
        bnx2x_ack_sb(bp, DEF_SB_ID, TSTORM_ID, le16_to_cpu(bp->def_t_idx),
                     IGU_INT_ENABLE, 1);

}

static irqreturn_t bnx2x_msix_sp_int(int irq, void *dev_instance)
{
        struct net_device *dev = dev_instance;
        struct bnx2x *bp = netdev_priv(dev);

        /* Return here if interrupt is disabled */
        if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
                DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
                return IRQ_HANDLED;
        }

        bnx2x_ack_sb(bp, DEF_SB_ID, XSTORM_ID, 0, IGU_INT_DISABLE, 0);

#ifdef BNX2X_STOP_ON_ERROR
        if (unlikely(bp->panic))
                return IRQ_HANDLED;
#endif

        schedule_work(&bp->sp_task);

        return IRQ_HANDLED;
}

/* end of slow path */

/* Statistics */

/****************************************************************************
* Macros
****************************************************************************/

/* sum[hi:lo] += add[hi:lo] */
#define ADD_64(s_hi, a_hi, s_lo, a_lo) \
        do { \
                s_lo += a_lo; \
                s_hi += a_hi + (s_lo < a_lo) ? 1 : 0; \
        } while (0)

/* difference = minuend - subtrahend */
#define DIFF_64(d_hi, m_hi, s_hi, d_lo, m_lo, s_lo) \
        do { \
                if (m_lo < s_lo) { \
                        /* underflow */ \
                        d_hi = m_hi - s_hi; \
                        if (d_hi > 0) { \
                                /* we can 'loan' 1 */ \
                                d_hi--; \
                                d_lo = m_lo + (UINT_MAX - s_lo) + 1; \
                        } else { \
                                /* m_hi <= s_hi */ \
                                d_hi = 0; \
                                d_lo = 0; \
                        } \
                } else { \
                        /* m_lo >= s_lo */ \
                        if (m_hi < s_hi) { \
                                d_hi = 0; \
                                d_lo = 0; \
                        } else { \
                                /* m_hi >= s_hi */ \
                                d_hi = m_hi - s_hi; \
                                d_lo = m_lo - s_lo; \
                        } \
                } \
        } while (0)

#define UPDATE_STAT64(s, t) \
        do { \
                DIFF_64(diff.hi, new->s##_hi, pstats->mac_stx[0].t##_hi, \
                        diff.lo, new->s##_lo, pstats->mac_stx[0].t##_lo); \
                pstats->mac_stx[0].t##_hi = new->s##_hi; \
                pstats->mac_stx[0].t##_lo = new->s##_lo; \
                ADD_64(pstats->mac_stx[1].t##_hi, diff.hi, \
                       pstats->mac_stx[1].t##_lo, diff.lo); \
        } while (0)

#define UPDATE_STAT64_NIG(s, t) \
        do { \
                DIFF_64(diff.hi, new->s##_hi, old->s##_hi, \
                        diff.lo, new->s##_lo, old->s##_lo); \
                ADD_64(estats->t##_hi, diff.hi, \
                       estats->t##_lo, diff.lo); \
        } while (0)

/* sum[hi:lo] += add */
#define ADD_EXTEND_64(s_hi, s_lo, a) \
        do { \
                s_lo += a; \
                s_hi += (s_lo < a) ? 1 : 0; \
        } while (0)

#define UPDATE_EXTEND_STAT(s) \
        do { \
                ADD_EXTEND_64(pstats->mac_stx[1].s##_hi, \
                              pstats->mac_stx[1].s##_lo, \
                              new->s); \
        } while (0)

#define UPDATE_EXTEND_TSTAT(s, t) \
        do { \
                diff = le32_to_cpu(tclient->s) - old_tclient->s; \
                old_tclient->s = le32_to_cpu(tclient->s); \
                ADD_EXTEND_64(fstats->t##_hi, fstats->t##_lo, diff); \
        } while (0)

#define UPDATE_EXTEND_XSTAT(s, t) \
        do { \
                diff = le32_to_cpu(xclient->s) - old_xclient->s; \
                old_xclient->s = le32_to_cpu(xclient->s); \
                ADD_EXTEND_64(fstats->t##_hi, fstats->t##_lo, diff); \
        } while (0)

/*
 * General service functions
 */

static inline long bnx2x_hilo(u32 *hiref)
{
        u32 lo = *(hiref + 1);
#if (BITS_PER_LONG == 64)
        u32 hi = *hiref;

        return HILO_U64(hi, lo);
#else
        return lo;
#endif
}

/*
 * Init service functions
 */

static void bnx2x_storm_stats_post(struct bnx2x *bp)
{
        if (!bp->stats_pending) {
                struct eth_query_ramrod_data ramrod_data = {0};
                int rc;