/*
 * Copyright(c) 2015 - 2020 Intel Corporation.
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * BSD LICENSE
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  - Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  - Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  - Neither the name of Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*
 * This file contains all of the code that is specific to the HFI chip
 */

#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/module.h>

#include "hfi.h"
#include "trace.h"
#include "mad.h"
#include "pio.h"
#include "sdma.h"
#include "eprom.h"
#include "efivar.h"
#include "platform.h"
#include "aspm.h"
#include "affinity.h"
#include "debugfs.h"
#include "fault.h"
#include "netdev.h"

uint num_vls = HFI1_MAX_VLS_SUPPORTED;
module_param(num_vls, uint, S_IRUGO);
MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");

/*
 * Default time to aggregate two 10K packets from the idle state
 * (timer not running). The timer starts at the end of the first packet,
 * so only the time for one 10K packet and header plus a bit extra is needed.
 * 10 * 1024 + 64 header byte = 10304 byte
 * 10304 byte / 12.5 GB/s = 824.32ns
 */
uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
module_param(rcv_intr_timeout, uint, S_IRUGO);
MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");

uint rcv_intr_count = 16; /* same as qib */
module_param(rcv_intr_count, uint, S_IRUGO);
MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");

ushort link_crc_mask = SUPPORTED_CRCS;
module_param(link_crc_mask, ushort, S_IRUGO);
MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");

uint loopback;
module_param_named(loopback, loopback, uint, S_IRUGO);
MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");

/* Other driver tunables */
uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
static ushort crc_14b_sideband = 1;
static uint use_flr = 1;
uint quick_linkup; /* skip LNI */

struct flag_table {
        u64 flag;       /* the flag */
        char *str;      /* description string */
        u16 extra;      /* extra information */
        u16 unused0;
        u32 unused1;
};

/* str must be a string constant */
#define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
#define FLAG_ENTRY0(str, flag) {flag, str, 0}

/* Send Error Consequences */
#define SEC_WRITE_DROPPED       0x1
#define SEC_PACKET_DROPPED      0x2
#define SEC_SC_HALTED           0x4     /* per-context only */
#define SEC_SPC_FREEZE          0x8     /* per-HFI only */

#define DEFAULT_KRCVQS            2
#define MIN_KERNEL_KCTXTS         2
#define FIRST_KERNEL_KCTXT        1

/*
 * RSM instance allocation
 *   0 - User Fecn Handling
 *   1 - Vnic
 *   2 - AIP
 *   3 - Verbs
 */
#define RSM_INS_FECN              0
#define RSM_INS_VNIC              1
#define RSM_INS_AIP               2
#define RSM_INS_VERBS             3

/* Bit offset into the GUID which carries HFI id information */
#define GUID_HFI_INDEX_SHIFT     39

/* extract the emulation revision */
#define emulator_rev(dd) ((dd)->irev >> 8)
/* parallel and serial emulation versions are 3 and 4 respectively */
#define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
#define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)

/* RSM fields for Verbs */
/* packet type */
#define IB_PACKET_TYPE         2ull
#define QW_SHIFT               6ull
/* QPN[7..1] */
#define QPN_WIDTH              7ull

/* LRH.BTH: QW 0, OFFSET 48 - for match */
#define LRH_BTH_QW             0ull
#define LRH_BTH_BIT_OFFSET     48ull
#define LRH_BTH_OFFSET(off)    ((LRH_BTH_QW << QW_SHIFT) | (off))
#define LRH_BTH_MATCH_OFFSET   LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
#define LRH_BTH_SELECT
#define LRH_BTH_MASK           3ull
#define LRH_BTH_VALUE          2ull

/* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
#define LRH_SC_QW              0ull
#define LRH_SC_BIT_OFFSET      56ull
#define LRH_SC_OFFSET(off)     ((LRH_SC_QW << QW_SHIFT) | (off))
#define LRH_SC_MATCH_OFFSET    LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
#define LRH_SC_MASK            128ull
#define LRH_SC_VALUE           0ull

/* SC[n..0] QW 0, OFFSET 60 - for select */
#define LRH_SC_SELECT_OFFSET  ((LRH_SC_QW << QW_SHIFT) | (60ull))

/* QPN[m+n:1] QW 1, OFFSET 1 */
#define QPN_SELECT_OFFSET      ((1ull << QW_SHIFT) | (1ull))

/* RSM fields for AIP */
/* LRH.BTH above is reused for this rule */

/* BTH.DESTQP: QW 1, OFFSET 16 for match */
#define BTH_DESTQP_QW           1ull
#define BTH_DESTQP_BIT_OFFSET   16ull
#define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW << QW_SHIFT) | (off))
#define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET)
#define BTH_DESTQP_MASK         0xFFull
#define BTH_DESTQP_VALUE        0x81ull

/* DETH.SQPN: QW 1 Offset 56 for select */
/* We use 8 most significant Soure QPN bits as entropy fpr AIP */
#define DETH_AIP_SQPN_QW 3ull
#define DETH_AIP_SQPN_BIT_OFFSET 56ull
#define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW << QW_SHIFT) | (off))
#define DETH_AIP_SQPN_SELECT_OFFSET \
        DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET)

/* RSM fields for Vnic */
/* L2_TYPE: QW 0, OFFSET 61 - for match */
#define L2_TYPE_QW             0ull
#define L2_TYPE_BIT_OFFSET     61ull
#define L2_TYPE_OFFSET(off)    ((L2_TYPE_QW << QW_SHIFT) | (off))
#define L2_TYPE_MATCH_OFFSET   L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET)
#define L2_TYPE_MASK           3ull
#define L2_16B_VALUE           2ull

/* L4_TYPE QW 1, OFFSET 0 - for match */
#define L4_TYPE_QW              1ull
#define L4_TYPE_BIT_OFFSET      0ull
#define L4_TYPE_OFFSET(off)     ((L4_TYPE_QW << QW_SHIFT) | (off))
#define L4_TYPE_MATCH_OFFSET    L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET)
#define L4_16B_TYPE_MASK        0xFFull
#define L4_16B_ETH_VALUE        0x78ull

/* 16B VESWID - for select */
#define L4_16B_HDR_VESWID_OFFSET  ((2 << QW_SHIFT) | (16ull))
/* 16B ENTROPY - for select */
#define L2_16B_ENTROPY_OFFSET     ((1 << QW_SHIFT) | (32ull))

/* defines to build power on SC2VL table */
#define SC2VL_VAL( \
        num, \
        sc0, sc0val, \
        sc1, sc1val, \
        sc2, sc2val, \
        sc3, sc3val, \
        sc4, sc4val, \
        sc5, sc5val, \
        sc6, sc6val, \
        sc7, sc7val) \
( \
        ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
        ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
        ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
        ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
        ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
        ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
        ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
        ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT)   \
)

#define DC_SC_VL_VAL( \
        range, \
        e0, e0val, \
        e1, e1val, \
        e2, e2val, \
        e3, e3val, \
        e4, e4val, \
        e5, e5val, \
        e6, e6val, \
        e7, e7val, \
        e8, e8val, \
        e9, e9val, \
        e10, e10val, \
        e11, e11val, \
        e12, e12val, \
        e13, e13val, \
        e14, e14val, \
        e15, e15val) \
( \
        ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
        ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
        ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
        ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
        ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
        ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
        ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
        ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
        ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
        ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
        ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
        ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
        ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
        ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
        ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
        ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
)

/* all CceStatus sub-block freeze bits */
#define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
                        | CCE_STATUS_RXE_FROZE_SMASK \
                        | CCE_STATUS_TXE_FROZE_SMASK \
                        | CCE_STATUS_TXE_PIO_FROZE_SMASK)
/* all CceStatus sub-block TXE pause bits */
#define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
                        | CCE_STATUS_TXE_PAUSED_SMASK \
                        | CCE_STATUS_SDMA_PAUSED_SMASK)
/* all CceStatus sub-block RXE pause bits */
#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK

#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
#define CNTR_32BIT_MAX 0x00000000FFFFFFFF

/*
 * CCE Error flags.
 */
static struct flag_table cce_err_status_flags[] = {
/* 0*/  FLAG_ENTRY0("CceCsrParityErr",
                CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
/* 1*/  FLAG_ENTRY0("CceCsrReadBadAddrErr",
                CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
/* 2*/  FLAG_ENTRY0("CceCsrWriteBadAddrErr",
                CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
/* 3*/  FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
                CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
/* 4*/  FLAG_ENTRY0("CceTrgtAccessErr",
                CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
/* 5*/  FLAG_ENTRY0("CceRspdDataParityErr",
                CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
/* 6*/  FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
                CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
/* 7*/  FLAG_ENTRY0("CceCsrCfgBusParityErr",
                CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
/* 8*/  FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
                CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
/* 9*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
            CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
/*10*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
            CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
/*11*/  FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
            CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
/*12*/  FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
                CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
/*13*/  FLAG_ENTRY0("PcicRetryMemCorErr",
                CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
/*14*/  FLAG_ENTRY0("PcicRetryMemCorErr",
                CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
/*15*/  FLAG_ENTRY0("PcicPostHdQCorErr",
                CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
/*16*/  FLAG_ENTRY0("PcicPostHdQCorErr",
                CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
/*17*/  FLAG_ENTRY0("PcicPostHdQCorErr",
                CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
/*18*/  FLAG_ENTRY0("PcicCplDatQCorErr",
                CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
/*19*/  FLAG_ENTRY0("PcicNPostHQParityErr",
                CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
/*20*/  FLAG_ENTRY0("PcicNPostDatQParityErr",
                CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
/*21*/  FLAG_ENTRY0("PcicRetryMemUncErr",
                CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
/*22*/  FLAG_ENTRY0("PcicRetrySotMemUncErr",
                CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
/*23*/  FLAG_ENTRY0("PcicPostHdQUncErr",
                CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
/*24*/  FLAG_ENTRY0("PcicPostDatQUncErr",
                CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
/*25*/  FLAG_ENTRY0("PcicCplHdQUncErr",
                CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
/*26*/  FLAG_ENTRY0("PcicCplDatQUncErr",
                CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
/*27*/  FLAG_ENTRY0("PcicTransmitFrontParityErr",
                CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
/*28*/  FLAG_ENTRY0("PcicTransmitBackParityErr",
                CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
/*29*/  FLAG_ENTRY0("PcicReceiveParityErr",
                CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
/*30*/  FLAG_ENTRY0("CceTrgtCplTimeoutErr",
                CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
/*31*/  FLAG_ENTRY0("LATriggered",
                CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
/*32*/  FLAG_ENTRY0("CceSegReadBadAddrErr",
                CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
/*33*/  FLAG_ENTRY0("CceSegWriteBadAddrErr",
                CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
/*34*/  FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
                CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
/*35*/  FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
                CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
/*36*/  FLAG_ENTRY0("CceMsixTableCorErr",
                CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
/*37*/  FLAG_ENTRY0("CceMsixTableUncErr",
                CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
/*38*/  FLAG_ENTRY0("CceIntMapCorErr",
                CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
/*39*/  FLAG_ENTRY0("CceIntMapUncErr",
                CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
/*40*/  FLAG_ENTRY0("CceMsixCsrParityErr",
                CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
/*41-63 reserved*/
};

/*
 * Misc Error flags
 */
#define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
static struct flag_table misc_err_status_flags[] = {
/* 0*/  FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
/* 1*/  FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
/* 2*/  FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
/* 3*/  FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
/* 4*/  FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
/* 5*/  FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
/* 6*/  FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
/* 7*/  FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
/* 8*/  FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
/* 9*/  FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
/*10*/  FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
/*11*/  FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
/*12*/  FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
};

/*
 * TXE PIO Error flags and consequences
 */
static struct flag_table pio_err_status_flags[] = {
/* 0*/  FLAG_ENTRY("PioWriteBadCtxt",
        SEC_WRITE_DROPPED,
        SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
/* 1*/  FLAG_ENTRY("PioWriteAddrParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
/* 2*/  FLAG_ENTRY("PioCsrParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
/* 3*/  FLAG_ENTRY("PioSbMemFifo0",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
/* 4*/  FLAG_ENTRY("PioSbMemFifo1",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
/* 5*/  FLAG_ENTRY("PioPccFifoParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
/* 6*/  FLAG_ENTRY("PioPecFifoParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
/* 7*/  FLAG_ENTRY("PioSbrdctlCrrelParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
/* 8*/  FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
/* 9*/  FLAG_ENTRY("PioPktEvictFifoParityErr",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
/*10*/  FLAG_ENTRY("PioSmPktResetParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
/*11*/  FLAG_ENTRY("PioVlLenMemBank0Unc",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
/*12*/  FLAG_ENTRY("PioVlLenMemBank1Unc",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
/*13*/  FLAG_ENTRY("PioVlLenMemBank0Cor",
        0,
        SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
/*14*/  FLAG_ENTRY("PioVlLenMemBank1Cor",
        0,
        SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
/*15*/  FLAG_ENTRY("PioCreditRetFifoParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
/*16*/  FLAG_ENTRY("PioPpmcPblFifo",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
/*17*/  FLAG_ENTRY("PioInitSmIn",
        0,
        SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
/*18*/  FLAG_ENTRY("PioPktEvictSmOrArbSm",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
/*19*/  FLAG_ENTRY("PioHostAddrMemUnc",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
/*20*/  FLAG_ENTRY("PioHostAddrMemCor",
        0,
        SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
/*21*/  FLAG_ENTRY("PioWriteDataParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
/*22*/  FLAG_ENTRY("PioStateMachine",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
/*23*/  FLAG_ENTRY("PioWriteQwValidParity",
        SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
/*24*/  FLAG_ENTRY("PioBlockQwCountParity",
        SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
/*25*/  FLAG_ENTRY("PioVlfVlLenParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
/*26*/  FLAG_ENTRY("PioVlfSopParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
/*27*/  FLAG_ENTRY("PioVlFifoParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
/*28*/  FLAG_ENTRY("PioPpmcBqcMemParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
/*29*/  FLAG_ENTRY("PioPpmcSopLen",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
/*30-31 reserved*/
/*32*/  FLAG_ENTRY("PioCurrentFreeCntParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
/*33*/  FLAG_ENTRY("PioLastReturnedCntParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
/*34*/  FLAG_ENTRY("PioPccSopHeadParity",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
/*35*/  FLAG_ENTRY("PioPecSopHeadParityErr",
        SEC_SPC_FREEZE,
        SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
/*36-63 reserved*/
};

/* TXE PIO errors that cause an SPC freeze */
#define ALL_PIO_FREEZE_ERR \
        (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
        | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)

/*
 * TXE SDMA Error flags
 */
static struct flag_table sdma_err_status_flags[] = {
/* 0*/  FLAG_ENTRY0("SDmaRpyTagErr",
                SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
/* 1*/  FLAG_ENTRY0("SDmaCsrParityErr",
                SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
/* 2*/  FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
                SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
/* 3*/  FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
                SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
/*04-63 reserved*/
};

/* TXE SDMA errors that cause an SPC freeze */
#define ALL_SDMA_FREEZE_ERR  \
                (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
                | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
                | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)

/* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
#define PORT_DISCARD_EGRESS_ERRS \
        (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
        | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
        | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)

/*
 * TXE Egress Error flags
 */
#define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
static struct flag_table egress_err_status_flags[] = {
/* 0*/  FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
/* 1*/  FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
/* 2 reserved */
/* 3*/  FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
                SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
/* 4*/  FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
/* 5*/  FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
/* 6 reserved */
/* 7*/  FLAG_ENTRY0("TxPioLaunchIntfParityErr",
                SEES(TX_PIO_LAUNCH_INTF_PARITY)),
/* 8*/  FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
                SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
/* 9-10 reserved */
/*11*/  FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
                SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
/*12*/  FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
/*13*/  FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
/*14*/  FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
/*15*/  FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
/*16*/  FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
                SEES(TX_SDMA0_DISALLOWED_PACKET)),
/*17*/  FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
                SEES(TX_SDMA1_DISALLOWED_PACKET)),
/*18*/  FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
                SEES(TX_SDMA2_DISALLOWED_PACKET)),
/*19*/  FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
                SEES(TX_SDMA3_DISALLOWED_PACKET)),
/*20*/  FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
                SEES(TX_SDMA4_DISALLOWED_PACKET)),
/*21*/  FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
                SEES(TX_SDMA5_DISALLOWED_PACKET)),
/*22*/  FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
                SEES(TX_SDMA6_DISALLOWED_PACKET)),
/*23*/  FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
                SEES(TX_SDMA7_DISALLOWED_PACKET)),
/*24*/  FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
                SEES(TX_SDMA8_DISALLOWED_PACKET)),
/*25*/  FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
                SEES(TX_SDMA9_DISALLOWED_PACKET)),
/*26*/  FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
                SEES(TX_SDMA10_DISALLOWED_PACKET)),
/*27*/  FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
                SEES(TX_SDMA11_DISALLOWED_PACKET)),
/*28*/  FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
                SEES(TX_SDMA12_DISALLOWED_PACKET)),
/*29*/  FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
                SEES(TX_SDMA13_DISALLOWED_PACKET)),
/*30*/  FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
                SEES(TX_SDMA14_DISALLOWED_PACKET)),
/*31*/  FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
                SEES(TX_SDMA15_DISALLOWED_PACKET)),
/*32*/  FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
                SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
/*33*/  FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
                SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
/*34*/  FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
                SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
/*35*/  FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
                SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
/*36*/  FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
                SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
/*37*/  FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
                SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
/*38*/  FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
                SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
/*39*/  FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
                SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
/*40*/  FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
                SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
/*41*/  FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
/*42*/  FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
/*43*/  FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
/*44*/  FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
/*45*/  FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
/*46*/  FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
/*47*/  FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
/*48*/  FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
/*49*/  FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
/*50*/  FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
/*51*/  FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
/*52*/  FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
/*53*/  FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
/*54*/  FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
/*55*/  FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
/*56*/  FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
/*57*/  FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
/*58*/  FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
/*59*/  FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
/*60*/  FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
/*61*/  FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
/*62*/  FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
                SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
/*63*/  FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
                SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
};

/*
 * TXE Egress Error Info flags
 */
#define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
static struct flag_table egress_err_info_flags[] = {
/* 0*/  FLAG_ENTRY0("Reserved", 0ull),
/* 1*/  FLAG_ENTRY0("VLErr", SEEI(VL)),
/* 2*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
/* 3*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
/* 4*/  FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
/* 5*/  FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
/* 6*/  FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
/* 7*/  FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
/* 8*/  FLAG_ENTRY0("RawErr", SEEI(RAW)),
/* 9*/  FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
/*10*/  FLAG_ENTRY0("GRHErr", SEEI(GRH)),
/*11*/  FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
/*12*/  FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
/*13*/  FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
/*14*/  FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
/*15*/  FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
/*16*/  FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
/*17*/  FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
/*18*/  FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
/*19*/  FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
/*20*/  FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
/*21*/  FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
};

/* TXE Egress errors that cause an SPC freeze */
#define ALL_TXE_EGRESS_FREEZE_ERR \
        (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
        | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
        | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
        | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
        | SEES(TX_LAUNCH_CSR_PARITY) \
        | SEES(TX_SBRD_CTL_CSR_PARITY) \
        | SEES(TX_CONFIG_PARITY) \
        | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
        | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
        | SEES(TX_CREDIT_RETURN_PARITY))

/*
 * TXE Send error flags
 */
#define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
static struct flag_table send_err_status_flags[] = {
/* 0*/  FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
/* 1*/  FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
/* 2*/  FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
};

/*
 * TXE Send Context Error flags and consequences
 */
static struct flag_table sc_err_status_flags[] = {
/* 0*/  FLAG_ENTRY("InconsistentSop",
                SEC_PACKET_DROPPED | SEC_SC_HALTED,
                SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
/* 1*/  FLAG_ENTRY("DisallowedPacket",
                SEC_PACKET_DROPPED | SEC_SC_HALTED,
                SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
/* 2*/  FLAG_ENTRY("WriteCrossesBoundary",
                SEC_WRITE_DROPPED | SEC_SC_HALTED,
                SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
/* 3*/  FLAG_ENTRY("WriteOverflow",
                SEC_WRITE_DROPPED | SEC_SC_HALTED,
                SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
/* 4*/  FLAG_ENTRY("WriteOutOfBounds",
                SEC_WRITE_DROPPED | SEC_SC_HALTED,
                SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
/* 5-63 reserved*/
};

/*
 * RXE Receive Error flags
 */
#define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
static struct flag_table rxe_err_status_flags[] = {
/* 0*/  FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
/* 1*/  FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
/* 2*/  FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
/* 3*/  FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
/* 4*/  FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
/* 5*/  FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
/* 6*/  FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
/* 7*/  FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
/* 8*/  FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
/* 9*/  FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
/*10*/  FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
/*11*/  FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
/*12*/  FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
/*13*/  FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
/*14*/  FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
/*15*/  FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
/*16*/  FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
                RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
/*17*/  FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
/*18*/  FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
/*19*/  FLAG_ENTRY0("RxRbufBlockListReadUncErr",
                RXES(RBUF_BLOCK_LIST_READ_UNC)),
/*20*/  FLAG_ENTRY0("RxRbufBlockListReadCorErr",
                RXES(RBUF_BLOCK_LIST_READ_COR)),
/*21*/  FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
                RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
/*22*/  FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
                RXES(RBUF_CSR_QENT_CNT_PARITY)),
/*23*/  FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
                RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
/*24*/  FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
                RXES(RBUF_CSR_QVLD_BIT_PARITY)),
/*25*/  FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
/*26*/  FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
/*27*/  FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
                RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
/*28*/  FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
/*29*/  FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
/*30*/  FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
/*31*/  FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
/*32*/  FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
/*33*/  FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
/*34*/  FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
/*35*/  FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
                RXES(RBUF_FL_INITDONE_PARITY)),
/*36*/  FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
                RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
/*37*/  FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
/*38*/  FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
/*39*/  FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
/*40*/  FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
                RXES(LOOKUP_DES_PART1_UNC_COR)),
/*41*/  FLAG_ENTRY0("RxLookupDesPart2ParityErr",
                RXES(LOOKUP_DES_PART2_PARITY)),
/*42*/  FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
/*43*/  FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
/*44*/  FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
/*45*/  FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
/*46*/  FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
/*47*/  FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
/*48*/  FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
/*49*/  FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
/*50*/  FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
/*51*/  FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
/*52*/  FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
/*53*/  FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
/*54*/  FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
/*55*/  FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
/*56*/  FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
/*57*/  FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
/*58*/  FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
/*59*/  FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
/*60*/  FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
/*61*/  FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
/*62*/  FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
/*63*/  FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
};

/* RXE errors that will trigger an SPC freeze */
#define ALL_RXE_FREEZE_ERR  \
        (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
        | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
        | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
        | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)

#define RXE_FREEZE_ABORT_MASK \
        (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
        RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
        RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)

/*
 * DCC Error Flags
 */
#define DCCE(name) DCC_ERR_FLG_##name##_SMASK
static struct flag_table dcc_err_flags[] = {
        FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
        FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
        FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
        FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
        FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
        FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
        FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
        FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
        FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
        FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
        FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
        FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
        FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
        FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
        FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
        FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
        FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
        FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
        FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
        FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
        FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
        FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
        FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
        FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
        FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
        FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
        FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
        FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
        FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
        FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
        FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
        FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
        FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
        FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
        FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
        FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
        FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
        FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
        FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
        FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
        FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
        FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
        FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
        FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
        FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
        FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
};

/*
 * LCB error flags
 */
#define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
static struct flag_table lcb_err_flags[] = {
/* 0*/  FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
/* 1*/  FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
/* 2*/  FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
/* 3*/  FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
                LCBE(ALL_LNS_FAILED_REINIT_TEST)),
/* 4*/  FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
/* 5*/  FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
/* 6*/  FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
/* 7*/  FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
/* 8*/  FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
/* 9*/  FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
/*10*/  FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
/*11*/  FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
/*12*/  FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
/*13*/  FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
                LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
/*14*/  FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
/*15*/  FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
/*16*/  FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
/*17*/  FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
/*18*/  FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
/*19*/  FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
                LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
/*20*/  FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
/*21*/  FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
/*22*/  FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
/*23*/  FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
/*24*/  FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
/*25*/  FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
/*26*/  FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
                LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
/*27*/  FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
/*28*/  FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
                LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
/*29*/  FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
                LCBE(REDUNDANT_FLIT_PARITY_ERR))
};

/*
 * DC8051 Error Flags
 */
#define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
static struct flag_table dc8051_err_flags[] = {
        FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
        FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
        FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
        FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
        FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
        FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
        FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
        FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
        FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
                    D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
        FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
};

/*
 * DC8051 Information Error flags
 *
 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
 */
static struct flag_table dc8051_info_err_flags[] = {
        FLAG_ENTRY0("Spico ROM check failed",  SPICO_ROM_FAILED),
        FLAG_ENTRY0("Unknown frame received",  UNKNOWN_FRAME),
        FLAG_ENTRY0("Target BER not met",      TARGET_BER_NOT_MET),
        FLAG_ENTRY0("Serdes internal loopback failure",
                    FAILED_SERDES_INTERNAL_LOOPBACK),
        FLAG_ENTRY0("Failed SerDes init",      FAILED_SERDES_INIT),
        FLAG_ENTRY0("Failed LNI(Polling)",     FAILED_LNI_POLLING),
        FLAG_ENTRY0("Failed LNI(Debounce)",    FAILED_LNI_DEBOUNCE),
        FLAG_ENTRY0("Failed LNI(EstbComm)",    FAILED_LNI_ESTBCOMM),
        FLAG_ENTRY0("Failed LNI(OptEq)",       FAILED_LNI_OPTEQ),
        FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
        FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
        FLAG_ENTRY0("Failed LNI(ConfigLT)",    FAILED_LNI_CONFIGLT),
        FLAG_ENTRY0("Host Handshake Timeout",  HOST_HANDSHAKE_TIMEOUT),
        FLAG_ENTRY0("External Device Request Timeout",
                    EXTERNAL_DEVICE_REQ_TIMEOUT),
};

/*
 * DC8051 Information Host Information flags
 *
 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
 */
static struct flag_table dc8051_info_host_msg_flags[] = {
        FLAG_ENTRY0("Host request done", 0x0001),
        FLAG_ENTRY0("BC PWR_MGM message", 0x0002),
        FLAG_ENTRY0("BC SMA message", 0x0004),
        FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
        FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
        FLAG_ENTRY0("External device config request", 0x0020),
        FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
        FLAG_ENTRY0("LinkUp achieved", 0x0080),
        FLAG_ENTRY0("Link going down", 0x0100),
        FLAG_ENTRY0("Link width downgraded", 0x0200),
};

static u32 encoded_size(u32 size);
static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
                               u8 *continuous);
static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
                                  u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
static void read_vc_remote_link_width(struct hfi1_devdata *dd,
                                      u8 *remote_tx_rate, u16 *link_widths);
static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
                                    u8 *flag_bits, u16 *link_widths);
static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
                                  u8 *device_rev);
static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
                            u8 *tx_polarity_inversion,
                            u8 *rx_polarity_inversion, u8 *max_rate);
static void handle_sdma_eng_err(struct hfi1_devdata *dd,
                                unsigned int context, u64 err_status);
static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
static void handle_dcc_err(struct hfi1_devdata *dd,
                           unsigned int context, u64 err_status);
static void handle_lcb_err(struct hfi1_devdata *dd,
                           unsigned int context, u64 err_status);
static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
static void set_partition_keys(struct hfi1_pportdata *ppd);
static const char *link_state_name(u32 state);
static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
                                          u32 state);
static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
                           u64 *out_data);
static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
static int thermal_init(struct hfi1_devdata *dd);

static void update_statusp(struct hfi1_pportdata *ppd, u32 state);
static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
                                            int msecs);
static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
                                  int msecs);
static void log_state_transition(struct hfi1_pportdata *ppd, u32 state);
static void log_physical_state(struct hfi1_pportdata *ppd, u32 state);
static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
                                   int msecs);
static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
                                         int msecs);
static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
static void handle_temp_err(struct hfi1_devdata *dd);
static void dc_shutdown(struct hfi1_devdata *dd);
static void dc_start(struct hfi1_devdata *dd);
static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
                           unsigned int *np);
static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);

/*
 * Error interrupt table entry.  This is used as input to the interrupt
 * "clear down" routine used for all second tier error interrupt register.
 * Second tier interrupt registers have a single bit representing them
 * in the top-level CceIntStatus.
 */
struct err_reg_info {
        u32 status;             /* status CSR offset */
        u32 clear;              /* clear CSR offset */
        u32 mask;               /* mask CSR offset */
        void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
        const char *desc;
};

#define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START)
#define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START)
#define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START)

/*
 * Helpers for building HFI and DC error interrupt table entries.  Different
 * helpers are needed because of inconsistent register names.
 */
#define EE(reg, handler, desc) \
        { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
                handler, desc }
#define DC_EE1(reg, handler, desc) \
        { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
#define DC_EE2(reg, handler, desc) \
        { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }

/*
 * Table of the "misc" grouping of error interrupts.  Each entry refers to
 * another register containing more information.
 */
static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
/* 0*/  EE(CCE_ERR,             handle_cce_err,    "CceErr"),
/* 1*/  EE(RCV_ERR,             handle_rxe_err,    "RxeErr"),
/* 2*/  EE(MISC_ERR,    handle_misc_err,   "MiscErr"),
/* 3*/  { 0, 0, 0, NULL }, /* reserved */
/* 4*/  EE(SEND_PIO_ERR,    handle_pio_err,    "PioErr"),
/* 5*/  EE(SEND_DMA_ERR,    handle_sdma_err,   "SDmaErr"),
/* 6*/  EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
/* 7*/  EE(SEND_ERR,    handle_txe_err,    "TxeErr")
        /* the rest are reserved */
};

/*
 * Index into the Various section of the interrupt sources
 * corresponding to the Critical Temperature interrupt.
 */
#define TCRIT_INT_SOURCE 4

/*
 * SDMA error interrupt entry - refers to another register containing more
 * information.
 */
static const struct err_reg_info sdma_eng_err =
        EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");

static const struct err_reg_info various_err[NUM_VARIOUS] = {
/* 0*/  { 0, 0, 0, NULL }, /* PbcInt */
/* 1*/  { 0, 0, 0, NULL }, /* GpioAssertInt */
/* 2*/  EE(ASIC_QSFP1,  handle_qsfp_int,        "QSFP1"),
/* 3*/  EE(ASIC_QSFP2,  handle_qsfp_int,        "QSFP2"),
/* 4*/  { 0, 0, 0, NULL }, /* TCritInt */
        /* rest are reserved */
};

/*
 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
 * register can not be derived from the MTU value because 10K is not
 * a power of 2. Therefore, we need a constant. Everything else can
 * be calculated.
 */
#define DCC_CFG_PORT_MTU_CAP_10240 7

/*
 * Table of the DC grouping of error interrupts.  Each entry refers to
 * another register containing more information.
 */
static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
/* 0*/  DC_EE1(DCC_ERR,         handle_dcc_err,        "DCC Err"),
/* 1*/  DC_EE2(DC_LCB_ERR,      handle_lcb_err,        "LCB Err"),
/* 2*/  DC_EE2(DC_DC8051_ERR,   handle_8051_interrupt, "DC8051 Interrupt"),
/* 3*/  /* dc_lbm_int - special, see is_dc_int() */
        /* the rest are reserved */
};

struct cntr_entry {
        /*
         * counter name
         */
        char *name;

        /*
         * csr to read for name (if applicable)
         */
        u64 csr;

        /*
         * offset into dd or ppd to store the counter's value
         */
        int offset;

        /*
         * flags
         */
        u8 flags;

        /*
         * accessor for stat element, context either dd or ppd
         */
        u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
                       int mode, u64 data);
};

#define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
#define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159

#define CNTR_ELEM(name, csr, offset, flags, accessor) \
{ \
        name, \
        csr, \
        offset, \
        flags, \
        accessor \
}

/* 32bit RXE */
#define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + RCV_COUNTER_ARRAY32), \
          0, flags | CNTR_32BIT, \
          port_access_u32_csr)

#define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + RCV_COUNTER_ARRAY32), \
          0, flags | CNTR_32BIT, \
          dev_access_u32_csr)

/* 64bit RXE */
#define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + RCV_COUNTER_ARRAY64), \
          0, flags, \
          port_access_u64_csr)

#define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + RCV_COUNTER_ARRAY64), \
          0, flags, \
          dev_access_u64_csr)

#define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
#define OVR_ELM(ctx) \
CNTR_ELEM("RcvHdrOvr" #ctx, \
          (RCV_HDR_OVFL_CNT + ctx * 0x100), \
          0, CNTR_NORMAL, port_access_u64_csr)

/* 32bit TXE */
#define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + SEND_COUNTER_ARRAY32), \
          0, flags | CNTR_32BIT, \
          port_access_u32_csr)

/* 64bit TXE */
#define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + SEND_COUNTER_ARRAY64), \
          0, flags, \
          port_access_u64_csr)

# define TX64_DEV_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name,\
          counter * 8 + SEND_COUNTER_ARRAY64, \
          0, \
          flags, \
          dev_access_u64_csr)

/* CCE */
#define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + CCE_COUNTER_ARRAY32), \
          0, flags | CNTR_32BIT, \
          dev_access_u32_csr)

#define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
CNTR_ELEM(#name, \
          (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
          0, flags | CNTR_32BIT, \
          dev_access_u32_csr)

/* DC */
#define DC_PERF_CNTR(name, counter, flags) \
CNTR_ELEM(#name, \
          counter, \
          0, \
          flags, \
          dev_access_u64_csr)

#define DC_PERF_CNTR_LCB(name, counter, flags) \
CNTR_ELEM(#name, \
          counter, \
          0, \
          flags, \
          dc_access_lcb_cntr)

/* ibp counters */
#define SW_IBP_CNTR(name, cntr) \
CNTR_ELEM(#name, \
          0, \
          0, \
          CNTR_SYNTH, \
          access_ibp_##cntr)

/**
 * hfi1_addr_from_offset - return addr for readq/writeq
 * @dd: the dd device
 * @offset: the offset of the CSR within bar0
 *
 * This routine selects the appropriate base address
 * based on the indicated offset.
 */
static inline void __iomem *hfi1_addr_from_offset(
        const struct hfi1_devdata *dd,
        u32 offset)
{
        if (offset >= dd->base2_start)
                return dd->kregbase2 + (offset - dd->base2_start);
        return dd->kregbase1 + offset;
}

/**
 * read_csr - read CSR at the indicated offset
 * @dd: the dd device
 * @offset: the offset of the CSR within bar0
 *
 * Return: the value read or all FF's if there
 * is no mapping
 */
u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
{
        if (dd->flags & HFI1_PRESENT)
                return readq(hfi1_addr_from_offset(dd, offset));
        return -1;
}

/**
 * write_csr - write CSR at the indicated offset
 * @dd: the dd device
 * @offset: the offset of the CSR within bar0
 * @value: value to write
 */
void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
{
        if (dd->flags & HFI1_PRESENT) {
                void __iomem *base = hfi1_addr_from_offset(dd, offset);

                /* avoid write to RcvArray */
                if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start))
                        return;
                writeq(value, base);
        }
}

/**
 * get_csr_addr - return te iomem address for offset
 * @dd: the dd device
 * @offset: the offset of the CSR within bar0
 *
 * Return: The iomem address to use in subsequent
 * writeq/readq operations.
 */
void __iomem *get_csr_addr(
        const struct hfi1_devdata *dd,
        u32 offset)
{
        if (dd->flags & HFI1_PRESENT)
                return hfi1_addr_from_offset(dd, offset);
        return NULL;
}

static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
                                 int mode, u64 value)
{
        u64 ret;

        if (mode == CNTR_MODE_R) {
                ret = read_csr(dd, csr);
        } else if (mode == CNTR_MODE_W) {
                write_csr(dd, csr, value);
                ret = value;
        } else {
                dd_dev_err(dd, "Invalid cntr register access mode");
                return 0;
        }

        hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
        return ret;
}

/* Dev Access */
static u64 dev_access_u32_csr(const struct cntr_entry *entry,
                              void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;
        u64 csr = entry->csr;

        if (entry->flags & CNTR_SDMA) {
                if (vl == CNTR_INVALID_VL)
                        return 0;
                csr += 0x100 * vl;
        } else {
                if (vl != CNTR_INVALID_VL)
                        return 0;
        }
        return read_write_csr(dd, csr, mode, data);
}

static u64 access_sde_err_cnt(const struct cntr_entry *entry,
                              void *context, int idx, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        if (dd->per_sdma && idx < dd->num_sdma)
                return dd->per_sdma[idx].err_cnt;
        return 0;
}

static u64 access_sde_int_cnt(const struct cntr_entry *entry,
                              void *context, int idx, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        if (dd->per_sdma && idx < dd->num_sdma)
                return dd->per_sdma[idx].sdma_int_cnt;
        return 0;
}

static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
                                   void *context, int idx, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        if (dd->per_sdma && idx < dd->num_sdma)
                return dd->per_sdma[idx].idle_int_cnt;
        return 0;
}

static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
                                       void *context, int idx, int mode,
                                       u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        if (dd->per_sdma && idx < dd->num_sdma)
                return dd->per_sdma[idx].progress_int_cnt;
        return 0;
}

static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
                              int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        u64 val = 0;
        u64 csr = entry->csr;

        if (entry->flags & CNTR_VL) {
                if (vl == CNTR_INVALID_VL)
                        return 0;
                csr += 8 * vl;
        } else {
                if (vl != CNTR_INVALID_VL)
                        return 0;
        }

        val = read_write_csr(dd, csr, mode, data);
        return val;
}

static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
                              int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;
        u32 csr = entry->csr;
        int ret = 0;

        if (vl != CNTR_INVALID_VL)
                return 0;
        if (mode == CNTR_MODE_R)
                ret = read_lcb_csr(dd, csr, &data);
        else if (mode == CNTR_MODE_W)
                ret = write_lcb_csr(dd, csr, data);

        if (ret) {
                dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
                return 0;
        }

        hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
        return data;
}

/* Port Access */
static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
                               int vl, int mode, u64 data)
{
        struct hfi1_pportdata *ppd = context;

        if (vl != CNTR_INVALID_VL)
                return 0;
        return read_write_csr(ppd->dd, entry->csr, mode, data);
}

static u64 port_access_u64_csr(const struct cntr_entry *entry,
                               void *context, int vl, int mode, u64 data)
{
        struct hfi1_pportdata *ppd = context;
        u64 val;
        u64 csr = entry->csr;

        if (entry->flags & CNTR_VL) {
                if (vl == CNTR_INVALID_VL)
                        return 0;
                csr += 8 * vl;
        } else {
                if (vl != CNTR_INVALID_VL)
                        return 0;
        }
        val = read_write_csr(ppd->dd, csr, mode, data);
        return val;
}

/* Software defined */
static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
                                u64 data)
{
        u64 ret;

        if (mode == CNTR_MODE_R) {
                ret = *cntr;
        } else if (mode == CNTR_MODE_W) {
                *cntr = data;
                ret = data;
        } else {
                dd_dev_err(dd, "Invalid cntr sw access mode");
                return 0;
        }

        hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);

        return ret;
}

static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
                                 int vl, int mode, u64 data)
{
        struct hfi1_pportdata *ppd = context;

        if (vl != CNTR_INVALID_VL)
                return 0;
        return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
}

static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
                                 int vl, int mode, u64 data)
{
        struct hfi1_pportdata *ppd = context;

        if (vl != CNTR_INVALID_VL)
                return 0;
        return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
}

static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
                                       void *context, int vl, int mode,
                                       u64 data)
{
        struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;

        if (vl != CNTR_INVALID_VL)
                return 0;
        return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
}

static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
                                   void *context, int vl, int mode, u64 data)
{
        struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
        u64 zero = 0;
        u64 *counter;

        if (vl == CNTR_INVALID_VL)
                counter = &ppd->port_xmit_discards;
        else if (vl >= 0 && vl < C_VL_COUNT)
                counter = &ppd->port_xmit_discards_vl[vl];
        else
                counter = &zero;

        return read_write_sw(ppd->dd, counter, mode, data);
}

static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
                                       void *context, int vl, int mode,
                                       u64 data)
{
        struct hfi1_pportdata *ppd = context;

        if (vl != CNTR_INVALID_VL)
                return 0;

        return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
                             mode, data);
}

static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
                                      void *context, int vl, int mode, u64 data)
{
        struct hfi1_pportdata *ppd = context;

        if (vl != CNTR_INVALID_VL)
                return 0;

        return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
                             mode, data);
}

u64 get_all_cpu_total(u64 __percpu *cntr)
{
        int cpu;
        u64 counter = 0;

        for_each_possible_cpu(cpu)
                counter += *per_cpu_ptr(cntr, cpu);
        return counter;
}

static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
                          u64 __percpu *cntr,
                          int vl, int mode, u64 data)
{
        u64 ret = 0;

        if (vl != CNTR_INVALID_VL)
                return 0;

        if (mode == CNTR_MODE_R) {
                ret = get_all_cpu_total(cntr) - *z_val;
        } else if (mode == CNTR_MODE_W) {
                /* A write can only zero the counter */
                if (data == 0)
                        *z_val = get_all_cpu_total(cntr);
                else
                        dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
        } else {
                dd_dev_err(dd, "Invalid cntr sw cpu access mode");
                return 0;
        }

        return ret;
}

static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
                              void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
                              mode, data);
}

static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
                                   void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
                              mode, data);
}

static u64 access_sw_pio_wait(const struct cntr_entry *entry,
                              void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        return dd->verbs_dev.n_piowait;
}

static u64 access_sw_pio_drain(const struct cntr_entry *entry,
                               void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->verbs_dev.n_piodrain;
}

static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry,
                                   void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        return dd->ctx0_seq_drop;
}

static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
                              void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        return dd->verbs_dev.n_txwait;
}

static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
                               void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = context;

        return dd->verbs_dev.n_kmem_wait;
}

static u64 access_sw_send_schedule(const struct cntr_entry *entry,
                                   void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
                              mode, data);
}

/* Software counters for the error status bits within MISC_ERR_STATUS */
static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
                                             void *context, int vl, int mode,
                                             u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[12];
}

static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
                                          void *context, int vl, int mode,
                                          u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[11];
}

static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
                                               void *context, int vl, int mode,
                                               u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[10];
}

static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
                                                 void *context, int vl,
                                                 int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[9];
}

static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
                                           void *context, int vl, int mode,
                                           u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[8];
}

static u64 access_misc_efuse_read_bad_addr_err_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[7];
}

static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
                                                void *context, int vl,
                                                int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[6];
}

static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
                                              void *context, int vl, int mode,
                                              u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[5];
}

static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
                                            void *context, int vl, int mode,
                                            u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[4];
}

static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
                                                 void *context, int vl,
                                                 int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[3];
}

static u64 access_misc_csr_write_bad_addr_err_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[2];
}

static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
                                                 void *context, int vl,
                                                 int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[1];
}

static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
                                          void *context, int vl, int mode,
                                          u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->misc_err_status_cnt[0];
}

/*
 * Software counter for the aggregate of
 * individual CceErrStatus counters
 */
static u64 access_sw_cce_err_status_aggregated_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->sw_cce_err_status_aggregate;
}

/*
 * Software counters corresponding to each of the
 * error status bits within CceErrStatus
 */
static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
                                              void *context, int vl, int mode,
                                              u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[40];
}

static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
                                          void *context, int vl, int mode,
                                          u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[39];
}

static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
                                          void *context, int vl, int mode,
                                          u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[38];
}

static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
                                             void *context, int vl, int mode,
                                             u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[37];
}

static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
                                             void *context, int vl, int mode,
                                             u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[36];
}

static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[35];
}

static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[34];
}

static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
                                                 void *context, int vl,
                                                 int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[33];
}

static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
                                                void *context, int vl, int mode,
                                                u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[32];
}

static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
                                   void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[31];
}

static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
                                               void *context, int vl, int mode,
                                               u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[30];
}

static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
                                              void *context, int vl, int mode,
                                              u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[29];
}

static u64 access_pcic_transmit_back_parity_err_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[28];
}

static u64 access_pcic_transmit_front_parity_err_cnt(
                                const struct cntr_entry *entry,
                                void *context, int vl, int mode, u64 data)
{
        struct hfi1_devdata *dd = (struct hfi1_devdata *)context;

        return dd->cce_err_status_cnt[27];
}

static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
                                             void *context, int vl, int mode,