// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * forcedeth: Ethernet driver for NVIDIA nForce media access controllers.
 *
 * Note: This driver is a cleanroom reimplementation based on reverse
 *      engineered documentation written by Carl-Daniel Hailfinger
 *      and Andrew de Quincey.
 *
 * NVIDIA, nForce and other NVIDIA marks are trademarks or registered
 * trademarks of NVIDIA Corporation in the United States and other
 * countries.
 *
 * Copyright (C) 2003,4,5 Manfred Spraul
 * Copyright (C) 2004 Andrew de Quincey (wol support)
 * Copyright (C) 2004 Carl-Daniel Hailfinger (invalid MAC handling, insane
 *              IRQ rate fixes, bigendian fixes, cleanups, verification)
 * Copyright (c) 2004,2005,2006,2007,2008,2009 NVIDIA Corporation
 *
 * Known bugs:
 * We suspect that on some hardware no TX done interrupts are generated.
 * This means recovery from netif_stop_queue only happens if the hw timer
 * interrupt fires (100 times/second, configurable with NVREG_POLL_DEFAULT)
 * and the timer is active in the IRQMask, or if a rx packet arrives by chance.
 * If your hardware reliably generates tx done interrupts, then you can remove
 * DEV_NEED_TIMERIRQ from the driver_data flags.
 * DEV_NEED_TIMERIRQ will not harm you on sane hardware, only generating a few
 * superfluous timer interrupts from the nic.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#define FORCEDETH_VERSION               "0.64"
#define DRV_NAME                        "forcedeth"

#include <linux/module.h>
#include <linux/types.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/delay.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/ethtool.h>
#include <linux/timer.h>
#include <linux/skbuff.h>
#include <linux/mii.h>
#include <linux/random.h>
#include <linux/if_vlan.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/prefetch.h>
#include <linux/u64_stats_sync.h>
#include <linux/io.h>

#include <asm/irq.h>

#define TX_WORK_PER_LOOP  64
#define RX_WORK_PER_LOOP  64

/*
 * Hardware access:
 */

#define DEV_NEED_TIMERIRQ          0x0000001  /* set the timer irq flag in the irq mask */
#define DEV_NEED_LINKTIMER         0x0000002  /* poll link settings. Relies on the timer irq */
#define DEV_HAS_LARGEDESC          0x0000004  /* device supports jumbo frames and needs packet format 2 */
#define DEV_HAS_HIGH_DMA           0x0000008  /* device supports 64bit dma */
#define DEV_HAS_CHECKSUM           0x0000010  /* device supports tx and rx checksum offloads */
#define DEV_HAS_VLAN               0x0000020  /* device supports vlan tagging and striping */
#define DEV_HAS_MSI                0x0000040  /* device supports MSI */
#define DEV_HAS_MSI_X              0x0000080  /* device supports MSI-X */
#define DEV_HAS_POWER_CNTRL        0x0000100  /* device supports power savings */
#define DEV_HAS_STATISTICS_V1      0x0000200  /* device supports hw statistics version 1 */
#define DEV_HAS_STATISTICS_V2      0x0000400  /* device supports hw statistics version 2 */
#define DEV_HAS_STATISTICS_V3      0x0000800  /* device supports hw statistics version 3 */
#define DEV_HAS_STATISTICS_V12     0x0000600  /* device supports hw statistics version 1 and 2 */
#define DEV_HAS_STATISTICS_V123    0x0000e00  /* device supports hw statistics version 1, 2, and 3 */
#define DEV_HAS_TEST_EXTENDED      0x0001000  /* device supports extended diagnostic test */
#define DEV_HAS_MGMT_UNIT          0x0002000  /* device supports management unit */
#define DEV_HAS_CORRECT_MACADDR    0x0004000  /* device supports correct mac address order */
#define DEV_HAS_COLLISION_FIX      0x0008000  /* device supports tx collision fix */
#define DEV_HAS_PAUSEFRAME_TX_V1   0x0010000  /* device supports tx pause frames version 1 */
#define DEV_HAS_PAUSEFRAME_TX_V2   0x0020000  /* device supports tx pause frames version 2 */
#define DEV_HAS_PAUSEFRAME_TX_V3   0x0040000  /* device supports tx pause frames version 3 */
#define DEV_NEED_TX_LIMIT          0x0080000  /* device needs to limit tx */
#define DEV_NEED_TX_LIMIT2         0x0180000  /* device needs to limit tx, expect for some revs */
#define DEV_HAS_GEAR_MODE          0x0200000  /* device supports gear mode */
#define DEV_NEED_PHY_INIT_FIX      0x0400000  /* device needs specific phy workaround */
#define DEV_NEED_LOW_POWER_FIX     0x0800000  /* device needs special power up workaround */
#define DEV_NEED_MSI_FIX           0x1000000  /* device needs msi workaround */

enum {
        NvRegIrqStatus = 0x000,
#define NVREG_IRQSTAT_MIIEVENT  0x040
#define NVREG_IRQSTAT_MASK              0x83ff
        NvRegIrqMask = 0x004,
#define NVREG_IRQ_RX_ERROR              0x0001
#define NVREG_IRQ_RX                    0x0002
#define NVREG_IRQ_RX_NOBUF              0x0004
#define NVREG_IRQ_TX_ERR                0x0008
#define NVREG_IRQ_TX_OK                 0x0010
#define NVREG_IRQ_TIMER                 0x0020
#define NVREG_IRQ_LINK                  0x0040
#define NVREG_IRQ_RX_FORCED             0x0080
#define NVREG_IRQ_TX_FORCED             0x0100
#define NVREG_IRQ_RECOVER_ERROR         0x8200
#define NVREG_IRQMASK_THROUGHPUT        0x00df
#define NVREG_IRQMASK_CPU               0x0060
#define NVREG_IRQ_TX_ALL                (NVREG_IRQ_TX_ERR|NVREG_IRQ_TX_OK|NVREG_IRQ_TX_FORCED)
#define NVREG_IRQ_RX_ALL                (NVREG_IRQ_RX_ERROR|NVREG_IRQ_RX|NVREG_IRQ_RX_NOBUF|NVREG_IRQ_RX_FORCED)
#define NVREG_IRQ_OTHER                 (NVREG_IRQ_TIMER|NVREG_IRQ_LINK|NVREG_IRQ_RECOVER_ERROR)

        NvRegUnknownSetupReg6 = 0x008,
#define NVREG_UNKSETUP6_VAL             3

/*
 * NVREG_POLL_DEFAULT is the interval length of the timer source on the nic
 * NVREG_POLL_DEFAULT=97 would result in an interval length of 1 ms
 */
        NvRegPollingInterval = 0x00c,
#define NVREG_POLL_DEFAULT_THROUGHPUT   65535 /* backup tx cleanup if loop max reached */
#define NVREG_POLL_DEFAULT_CPU  13
        NvRegMSIMap0 = 0x020,
        NvRegMSIMap1 = 0x024,
        NvRegMSIIrqMask = 0x030,
#define NVREG_MSI_VECTOR_0_ENABLED 0x01
        NvRegMisc1 = 0x080,
#define NVREG_MISC1_PAUSE_TX    0x01
#define NVREG_MISC1_HD          0x02
#define NVREG_MISC1_FORCE       0x3b0f3c

        NvRegMacReset = 0x34,
#define NVREG_MAC_RESET_ASSERT  0x0F3
        NvRegTransmitterControl = 0x084,
#define NVREG_XMITCTL_START     0x01
#define NVREG_XMITCTL_MGMT_ST   0x40000000
#define NVREG_XMITCTL_SYNC_MASK         0x000f0000
#define NVREG_XMITCTL_SYNC_NOT_READY    0x0
#define NVREG_XMITCTL_SYNC_PHY_INIT     0x00040000
#define NVREG_XMITCTL_MGMT_SEMA_MASK    0x00000f00
#define NVREG_XMITCTL_MGMT_SEMA_FREE    0x0
#define NVREG_XMITCTL_HOST_SEMA_MASK    0x0000f000
#define NVREG_XMITCTL_HOST_SEMA_ACQ     0x0000f000
#define NVREG_XMITCTL_HOST_LOADED       0x00004000
#define NVREG_XMITCTL_TX_PATH_EN        0x01000000
#define NVREG_XMITCTL_DATA_START        0x00100000
#define NVREG_XMITCTL_DATA_READY        0x00010000
#define NVREG_XMITCTL_DATA_ERROR        0x00020000
        NvRegTransmitterStatus = 0x088,
#define NVREG_XMITSTAT_BUSY     0x01

        NvRegPacketFilterFlags = 0x8c,
#define NVREG_PFF_PAUSE_RX      0x08
#define NVREG_PFF_ALWAYS        0x7F0000
#define NVREG_PFF_PROMISC       0x80
#define NVREG_PFF_MYADDR        0x20
#define NVREG_PFF_LOOPBACK      0x10

        NvRegOffloadConfig = 0x90,
#define NVREG_OFFLOAD_HOMEPHY   0x601
#define NVREG_OFFLOAD_NORMAL    RX_NIC_BUFSIZE
        NvRegReceiverControl = 0x094,
#define NVREG_RCVCTL_START      0x01
#define NVREG_RCVCTL_RX_PATH_EN 0x01000000
        NvRegReceiverStatus = 0x98,
#define NVREG_RCVSTAT_BUSY      0x01

        NvRegSlotTime = 0x9c,
#define NVREG_SLOTTIME_LEGBF_ENABLED    0x80000000
#define NVREG_SLOTTIME_10_100_FULL      0x00007f00
#define NVREG_SLOTTIME_1000_FULL        0x0003ff00
#define NVREG_SLOTTIME_HALF             0x0000ff00
#define NVREG_SLOTTIME_DEFAULT          0x00007f00
#define NVREG_SLOTTIME_MASK             0x000000ff

        NvRegTxDeferral = 0xA0,
#define NVREG_TX_DEFERRAL_DEFAULT               0x15050f
#define NVREG_TX_DEFERRAL_RGMII_10_100          0x16070f
#define NVREG_TX_DEFERRAL_RGMII_1000            0x14050f
#define NVREG_TX_DEFERRAL_RGMII_STRETCH_10      0x16190f
#define NVREG_TX_DEFERRAL_RGMII_STRETCH_100     0x16300f
#define NVREG_TX_DEFERRAL_MII_STRETCH           0x152000
        NvRegRxDeferral = 0xA4,
#define NVREG_RX_DEFERRAL_DEFAULT       0x16
        NvRegMacAddrA = 0xA8,
        NvRegMacAddrB = 0xAC,
        NvRegMulticastAddrA = 0xB0,
#define NVREG_MCASTADDRA_FORCE  0x01
        NvRegMulticastAddrB = 0xB4,
        NvRegMulticastMaskA = 0xB8,
#define NVREG_MCASTMASKA_NONE           0xffffffff
        NvRegMulticastMaskB = 0xBC,
#define NVREG_MCASTMASKB_NONE           0xffff

        NvRegPhyInterface = 0xC0,
#define PHY_RGMII               0x10000000
        NvRegBackOffControl = 0xC4,
#define NVREG_BKOFFCTRL_DEFAULT                 0x70000000
#define NVREG_BKOFFCTRL_SEED_MASK               0x000003ff
#define NVREG_BKOFFCTRL_SELECT                  24
#define NVREG_BKOFFCTRL_GEAR                    12

        NvRegTxRingPhysAddr = 0x100,
        NvRegRxRingPhysAddr = 0x104,
        NvRegRingSizes = 0x108,
#define NVREG_RINGSZ_TXSHIFT 0
#define NVREG_RINGSZ_RXSHIFT 16
        NvRegTransmitPoll = 0x10c,
#define NVREG_TRANSMITPOLL_MAC_ADDR_REV 0x00008000
        NvRegLinkSpeed = 0x110,
#define NVREG_LINKSPEED_FORCE 0x10000
#define NVREG_LINKSPEED_10      1000
#define NVREG_LINKSPEED_100     100
#define NVREG_LINKSPEED_1000    50
#define NVREG_LINKSPEED_MASK    (0xFFF)
        NvRegUnknownSetupReg5 = 0x130,
#define NVREG_UNKSETUP5_BIT31   (1<<31)
        NvRegTxWatermark = 0x13c,
#define NVREG_TX_WM_DESC1_DEFAULT       0x0200010
#define NVREG_TX_WM_DESC2_3_DEFAULT     0x1e08000
#define NVREG_TX_WM_DESC2_3_1000        0xfe08000
        NvRegTxRxControl = 0x144,
#define NVREG_TXRXCTL_KICK      0x0001
#define NVREG_TXRXCTL_BIT1      0x0002
#define NVREG_TXRXCTL_BIT2      0x0004
#define NVREG_TXRXCTL_IDLE      0x0008
#define NVREG_TXRXCTL_RESET     0x0010
#define NVREG_TXRXCTL_RXCHECK   0x0400
#define NVREG_TXRXCTL_DESC_1    0
#define NVREG_TXRXCTL_DESC_2    0x002100
#define NVREG_TXRXCTL_DESC_3    0xc02200
#define NVREG_TXRXCTL_VLANSTRIP 0x00040
#define NVREG_TXRXCTL_VLANINS   0x00080
        NvRegTxRingPhysAddrHigh = 0x148,
        NvRegRxRingPhysAddrHigh = 0x14C,
        NvRegTxPauseFrame = 0x170,
#define NVREG_TX_PAUSEFRAME_DISABLE     0x0fff0080
#define NVREG_TX_PAUSEFRAME_ENABLE_V1   0x01800010
#define NVREG_TX_PAUSEFRAME_ENABLE_V2   0x056003f0
#define NVREG_TX_PAUSEFRAME_ENABLE_V3   0x09f00880
        NvRegTxPauseFrameLimit = 0x174,
#define NVREG_TX_PAUSEFRAMELIMIT_ENABLE 0x00010000
        NvRegMIIStatus = 0x180,
#define NVREG_MIISTAT_ERROR             0x0001
#define NVREG_MIISTAT_LINKCHANGE        0x0008
#define NVREG_MIISTAT_MASK_RW           0x0007
#define NVREG_MIISTAT_MASK_ALL          0x000f
        NvRegMIIMask = 0x184,
#define NVREG_MII_LINKCHANGE            0x0008

        NvRegAdapterControl = 0x188,
#define NVREG_ADAPTCTL_START    0x02
#define NVREG_ADAPTCTL_LINKUP   0x04
#define NVREG_ADAPTCTL_PHYVALID 0x40000
#define NVREG_ADAPTCTL_RUNNING  0x100000
#define NVREG_ADAPTCTL_PHYSHIFT 24
        NvRegMIISpeed = 0x18c,
#define NVREG_MIISPEED_BIT8     (1<<8)
#define NVREG_MIIDELAY  5
        NvRegMIIControl = 0x190,
#define NVREG_MIICTL_INUSE      0x08000
#define NVREG_MIICTL_WRITE      0x00400
#define NVREG_MIICTL_ADDRSHIFT  5
        NvRegMIIData = 0x194,
        NvRegTxUnicast = 0x1a0,
        NvRegTxMulticast = 0x1a4,
        NvRegTxBroadcast = 0x1a8,
        NvRegWakeUpFlags = 0x200,
#define NVREG_WAKEUPFLAGS_VAL           0x7770
#define NVREG_WAKEUPFLAGS_BUSYSHIFT     24
#define NVREG_WAKEUPFLAGS_ENABLESHIFT   16
#define NVREG_WAKEUPFLAGS_D3SHIFT       12
#define NVREG_WAKEUPFLAGS_D2SHIFT       8
#define NVREG_WAKEUPFLAGS_D1SHIFT       4
#define NVREG_WAKEUPFLAGS_D0SHIFT       0
#define NVREG_WAKEUPFLAGS_ACCEPT_MAGPAT         0x01
#define NVREG_WAKEUPFLAGS_ACCEPT_WAKEUPPAT      0x02
#define NVREG_WAKEUPFLAGS_ACCEPT_LINKCHANGE     0x04
#define NVREG_WAKEUPFLAGS_ENABLE        0x1111

        NvRegMgmtUnitGetVersion = 0x204,
#define NVREG_MGMTUNITGETVERSION        0x01
        NvRegMgmtUnitVersion = 0x208,
#define NVREG_MGMTUNITVERSION           0x08
        NvRegPowerCap = 0x268,
#define NVREG_POWERCAP_D3SUPP   (1<<30)
#define NVREG_POWERCAP_D2SUPP   (1<<26)
#define NVREG_POWERCAP_D1SUPP   (1<<25)
        NvRegPowerState = 0x26c,
#define NVREG_POWERSTATE_POWEREDUP      0x8000
#define NVREG_POWERSTATE_VALID          0x0100
#define NVREG_POWERSTATE_MASK           0x0003
#define NVREG_POWERSTATE_D0             0x0000
#define NVREG_POWERSTATE_D1             0x0001
#define NVREG_POWERSTATE_D2             0x0002
#define NVREG_POWERSTATE_D3             0x0003
        NvRegMgmtUnitControl = 0x278,
#define NVREG_MGMTUNITCONTROL_INUSE     0x20000
        NvRegTxCnt = 0x280,
        NvRegTxZeroReXmt = 0x284,
        NvRegTxOneReXmt = 0x288,
        NvRegTxManyReXmt = 0x28c,
        NvRegTxLateCol = 0x290,
        NvRegTxUnderflow = 0x294,
        NvRegTxLossCarrier = 0x298,
        NvRegTxExcessDef = 0x29c,
        NvRegTxRetryErr = 0x2a0,
        NvRegRxFrameErr = 0x2a4,
        NvRegRxExtraByte = 0x2a8,
        NvRegRxLateCol = 0x2ac,
        NvRegRxRunt = 0x2b0,
        NvRegRxFrameTooLong = 0x2b4,
        NvRegRxOverflow = 0x2b8,
        NvRegRxFCSErr = 0x2bc,
        NvRegRxFrameAlignErr = 0x2c0,
        NvRegRxLenErr = 0x2c4,
        NvRegRxUnicast = 0x2c8,
        NvRegRxMulticast = 0x2cc,
        NvRegRxBroadcast = 0x2d0,
        NvRegTxDef = 0x2d4,
        NvRegTxFrame = 0x2d8,
        NvRegRxCnt = 0x2dc,
        NvRegTxPause = 0x2e0,
        NvRegRxPause = 0x2e4,
        NvRegRxDropFrame = 0x2e8,
        NvRegVlanControl = 0x300,
#define NVREG_VLANCONTROL_ENABLE        0x2000
        NvRegMSIXMap0 = 0x3e0,
        NvRegMSIXMap1 = 0x3e4,
        NvRegMSIXIrqStatus = 0x3f0,

        NvRegPowerState2 = 0x600,
#define NVREG_POWERSTATE2_POWERUP_MASK          0x0F15
#define NVREG_POWERSTATE2_POWERUP_REV_A3        0x0001
#define NVREG_POWERSTATE2_PHY_RESET             0x0004
#define NVREG_POWERSTATE2_GATE_CLOCKS           0x0F00
};

/* Big endian: should work, but is untested */
struct ring_desc {
        __le32 buf;
        __le32 flaglen;
};

struct ring_desc_ex {
        __le32 bufhigh;
        __le32 buflow;
        __le32 txvlan;
        __le32 flaglen;
};

union ring_type {
        struct ring_desc *orig;
        struct ring_desc_ex *ex;
};

#define FLAG_MASK_V1 0xffff0000
#define FLAG_MASK_V2 0xffffc000
#define LEN_MASK_V1 (0xffffffff ^ FLAG_MASK_V1)
#define LEN_MASK_V2 (0xffffffff ^ FLAG_MASK_V2)

#define NV_TX_LASTPACKET        (1<<16)
#define NV_TX_RETRYERROR        (1<<19)
#define NV_TX_RETRYCOUNT_MASK   (0xF<<20)
#define NV_TX_FORCED_INTERRUPT  (1<<24)
#define NV_TX_DEFERRED          (1<<26)
#define NV_TX_CARRIERLOST       (1<<27)
#define NV_TX_LATECOLLISION     (1<<28)
#define NV_TX_UNDERFLOW         (1<<29)
#define NV_TX_ERROR             (1<<30)
#define NV_TX_VALID             (1<<31)

#define NV_TX2_LASTPACKET       (1<<29)
#define NV_TX2_RETRYERROR       (1<<18)
#define NV_TX2_RETRYCOUNT_MASK  (0xF<<19)
#define NV_TX2_FORCED_INTERRUPT (1<<30)
#define NV_TX2_DEFERRED         (1<<25)
#define NV_TX2_CARRIERLOST      (1<<26)
#define NV_TX2_LATECOLLISION    (1<<27)
#define NV_TX2_UNDERFLOW        (1<<28)
/* error and valid are the same for both */
#define NV_TX2_ERROR            (1<<30)
#define NV_TX2_VALID            (1<<31)
#define NV_TX2_TSO              (1<<28)
#define NV_TX2_TSO_SHIFT        14
#define NV_TX2_TSO_MAX_SHIFT    14
#define NV_TX2_TSO_MAX_SIZE     (1<<NV_TX2_TSO_MAX_SHIFT)
#define NV_TX2_CHECKSUM_L3      (1<<27)
#define NV_TX2_CHECKSUM_L4      (1<<26)

#define NV_TX3_VLAN_TAG_PRESENT (1<<18)

#define NV_RX_DESCRIPTORVALID   (1<<16)
#define NV_RX_MISSEDFRAME       (1<<17)
#define NV_RX_SUBTRACT1         (1<<18)
#define NV_RX_ERROR1            (1<<23)
#define NV_RX_ERROR2            (1<<24)
#define NV_RX_ERROR3            (1<<25)
#define NV_RX_ERROR4            (1<<26)
#define NV_RX_CRCERR            (1<<27)
#define NV_RX_OVERFLOW          (1<<28)
#define NV_RX_FRAMINGERR        (1<<29)
#define NV_RX_ERROR             (1<<30)
#define NV_RX_AVAIL             (1<<31)
#define NV_RX_ERROR_MASK        (NV_RX_ERROR1|NV_RX_ERROR2|NV_RX_ERROR3|NV_RX_ERROR4|NV_RX_CRCERR|NV_RX_OVERFLOW|NV_RX_FRAMINGERR)

#define NV_RX2_CHECKSUMMASK     (0x1C000000)
#define NV_RX2_CHECKSUM_IP      (0x10000000)
#define NV_RX2_CHECKSUM_IP_TCP  (0x14000000)
#define NV_RX2_CHECKSUM_IP_UDP  (0x18000000)
#define NV_RX2_DESCRIPTORVALID  (1<<29)
#define NV_RX2_SUBTRACT1        (1<<25)
#define NV_RX2_ERROR1           (1<<18)
#define NV_RX2_ERROR2           (1<<19)
#define NV_RX2_ERROR3           (1<<20)
#define NV_RX2_ERROR4           (1<<21)
#define NV_RX2_CRCERR           (1<<22)
#define NV_RX2_OVERFLOW         (1<<23)
#define NV_RX2_FRAMINGERR       (1<<24)
/* error and avail are the same for both */
#define NV_RX2_ERROR            (1<<30)
#define NV_RX2_AVAIL            (1<<31)
#define NV_RX2_ERROR_MASK       (NV_RX2_ERROR1|NV_RX2_ERROR2|NV_RX2_ERROR3|NV_RX2_ERROR4|NV_RX2_CRCERR|NV_RX2_OVERFLOW|NV_RX2_FRAMINGERR)

#define NV_RX3_VLAN_TAG_PRESENT (1<<16)
#define NV_RX3_VLAN_TAG_MASK    (0x0000FFFF)

/* Miscellaneous hardware related defines: */
#define NV_PCI_REGSZ_VER1       0x270
#define NV_PCI_REGSZ_VER2       0x2d4
#define NV_PCI_REGSZ_VER3       0x604
#define NV_PCI_REGSZ_MAX        0x604

/* various timeout delays: all in usec */
#define NV_TXRX_RESET_DELAY     4
#define NV_TXSTOP_DELAY1        10
#define NV_TXSTOP_DELAY1MAX     500000
#define NV_TXSTOP_DELAY2        100
#define NV_RXSTOP_DELAY1        10
#define NV_RXSTOP_DELAY1MAX     500000
#define NV_RXSTOP_DELAY2        100
#define NV_SETUP5_DELAY         5
#define NV_SETUP5_DELAYMAX      50000
#define NV_POWERUP_DELAY        5
#define NV_POWERUP_DELAYMAX     5000
#define NV_MIIBUSY_DELAY        50
#define NV_MIIPHY_DELAY 10
#define NV_MIIPHY_DELAYMAX      10000
#define NV_MAC_RESET_DELAY      64

#define NV_WAKEUPPATTERNS       5
#define NV_WAKEUPMASKENTRIES    4

/* General driver defaults */
#define NV_WATCHDOG_TIMEO       (5*HZ)

#define RX_RING_DEFAULT         512
#define TX_RING_DEFAULT         256
#define RX_RING_MIN             128
#define TX_RING_MIN             64
#define RING_MAX_DESC_VER_1     1024
#define RING_MAX_DESC_VER_2_3   16384

/* rx/tx mac addr + type + vlan + align + slack*/
#define NV_RX_HEADERS           (64)
/* even more slack. */
#define NV_RX_ALLOC_PAD         (64)

/* maximum mtu size */
#define NV_PKTLIMIT_1   ETH_DATA_LEN    /* hard limit not known */
#define NV_PKTLIMIT_2   9100    /* Actual limit according to NVidia: 9202 */

#define OOM_REFILL      (1+HZ/20)
#define POLL_WAIT       (1+HZ/100)
#define LINK_TIMEOUT    (3*HZ)
#define STATS_INTERVAL  (10*HZ)

/*
 * desc_ver values:
 * The nic supports three different descriptor types:
 * - DESC_VER_1: Original
 * - DESC_VER_2: support for jumbo frames.
 * - DESC_VER_3: 64-bit format.
 */
#define DESC_VER_1      1
#define DESC_VER_2      2
#define DESC_VER_3      3

/* PHY defines */
#define PHY_OUI_MARVELL         0x5043
#define PHY_OUI_CICADA          0x03f1
#define PHY_OUI_VITESSE         0x01c1
#define PHY_OUI_REALTEK         0x0732
#define PHY_OUI_REALTEK2        0x0020
#define PHYID1_OUI_MASK 0x03ff
#define PHYID1_OUI_SHFT 6
#define PHYID2_OUI_MASK 0xfc00
#define PHYID2_OUI_SHFT 10
#define PHYID2_MODEL_MASK               0x03f0
#define PHY_MODEL_REALTEK_8211          0x0110
#define PHY_REV_MASK                    0x0001
#define PHY_REV_REALTEK_8211B           0x0000
#define PHY_REV_REALTEK_8211C           0x0001
#define PHY_MODEL_REALTEK_8201          0x0200
#define PHY_MODEL_MARVELL_E3016         0x0220
#define PHY_MARVELL_E3016_INITMASK      0x0300
#define PHY_CICADA_INIT1        0x0f000
#define PHY_CICADA_INIT2        0x0e00
#define PHY_CICADA_INIT3        0x01000
#define PHY_CICADA_INIT4        0x0200
#define PHY_CICADA_INIT5        0x0004
#define PHY_CICADA_INIT6        0x02000
#define PHY_VITESSE_INIT_REG1   0x1f
#define PHY_VITESSE_INIT_REG2   0x10
#define PHY_VITESSE_INIT_REG3   0x11
#define PHY_VITESSE_INIT_REG4   0x12
#define PHY_VITESSE_INIT_MSK1   0xc
#define PHY_VITESSE_INIT_MSK2   0x0180
#define PHY_VITESSE_INIT1       0x52b5
#define PHY_VITESSE_INIT2       0xaf8a
#define PHY_VITESSE_INIT3       0x8
#define PHY_VITESSE_INIT4       0x8f8a
#define PHY_VITESSE_INIT5       0xaf86
#define PHY_VITESSE_INIT6       0x8f86
#define PHY_VITESSE_INIT7       0xaf82
#define PHY_VITESSE_INIT8       0x0100
#define PHY_VITESSE_INIT9       0x8f82
#define PHY_VITESSE_INIT10      0x0
#define PHY_REALTEK_INIT_REG1   0x1f
#define PHY_REALTEK_INIT_REG2   0x19
#define PHY_REALTEK_INIT_REG3   0x13
#define PHY_REALTEK_INIT_REG4   0x14
#define PHY_REALTEK_INIT_REG5   0x18
#define PHY_REALTEK_INIT_REG6   0x11
#define PHY_REALTEK_INIT_REG7   0x01
#define PHY_REALTEK_INIT1       0x0000
#define PHY_REALTEK_INIT2       0x8e00
#define PHY_REALTEK_INIT3       0x0001
#define PHY_REALTEK_INIT4       0xad17
#define PHY_REALTEK_INIT5       0xfb54
#define PHY_REALTEK_INIT6       0xf5c7
#define PHY_REALTEK_INIT7       0x1000
#define PHY_REALTEK_INIT8       0x0003
#define PHY_REALTEK_INIT9       0x0008
#define PHY_REALTEK_INIT10      0x0005
#define PHY_REALTEK_INIT11      0x0200
#define PHY_REALTEK_INIT_MSK1   0x0003

#define PHY_GIGABIT     0x0100

#define PHY_TIMEOUT     0x1
#define PHY_ERROR       0x2

#define PHY_100 0x1
#define PHY_1000        0x2
#define PHY_HALF        0x100

#define NV_PAUSEFRAME_RX_CAPABLE 0x0001
#define NV_PAUSEFRAME_TX_CAPABLE 0x0002
#define NV_PAUSEFRAME_RX_ENABLE  0x0004
#define NV_PAUSEFRAME_TX_ENABLE  0x0008
#define NV_PAUSEFRAME_RX_REQ     0x0010
#define NV_PAUSEFRAME_TX_REQ     0x0020
#define NV_PAUSEFRAME_AUTONEG    0x0040

/* MSI/MSI-X defines */
#define NV_MSI_X_MAX_VECTORS  8
#define NV_MSI_X_VECTORS_MASK 0x000f
#define NV_MSI_CAPABLE        0x0010
#define NV_MSI_X_CAPABLE      0x0020
#define NV_MSI_ENABLED        0x0040
#define NV_MSI_X_ENABLED      0x0080

#define NV_MSI_X_VECTOR_ALL   0x0
#define NV_MSI_X_VECTOR_RX    0x0
#define NV_MSI_X_VECTOR_TX    0x1
#define NV_MSI_X_VECTOR_OTHER 0x2

#define NV_MSI_PRIV_OFFSET 0x68
#define NV_MSI_PRIV_VALUE  0xffffffff

#define NV_RESTART_TX         0x1
#define NV_RESTART_RX         0x2

#define NV_TX_LIMIT_COUNT     16

#define NV_DYNAMIC_THRESHOLD        4
#define NV_DYNAMIC_MAX_QUIET_COUNT  2048

/* statistics */
struct nv_ethtool_str {
        char name[ETH_GSTRING_LEN];
};

static const struct nv_ethtool_str nv_estats_str[] = {
        { "tx_bytes" }, /* includes Ethernet FCS CRC */
        { "tx_zero_rexmt" },
        { "tx_one_rexmt" },
        { "tx_many_rexmt" },
        { "tx_late_collision" },
        { "tx_fifo_errors" },
        { "tx_carrier_errors" },
        { "tx_excess_deferral" },
        { "tx_retry_error" },
        { "rx_frame_error" },
        { "rx_extra_byte" },
        { "rx_late_collision" },
        { "rx_runt" },
        { "rx_frame_too_long" },
        { "rx_over_errors" },
        { "rx_crc_errors" },
        { "rx_frame_align_error" },
        { "rx_length_error" },
        { "rx_unicast" },
        { "rx_multicast" },
        { "rx_broadcast" },
        { "rx_packets" },
        { "rx_errors_total" },
        { "tx_errors_total" },

        /* version 2 stats */
        { "tx_deferral" },
        { "tx_packets" },
        { "rx_bytes" }, /* includes Ethernet FCS CRC */
        { "tx_pause" },
        { "rx_pause" },
        { "rx_drop_frame" },

        /* version 3 stats */
        { "tx_unicast" },
        { "tx_multicast" },
        { "tx_broadcast" }
};

struct nv_ethtool_stats {
        u64 tx_bytes; /* should be ifconfig->tx_bytes + 4*tx_packets */
        u64 tx_zero_rexmt;
        u64 tx_one_rexmt;
        u64 tx_many_rexmt;
        u64 tx_late_collision;
        u64 tx_fifo_errors;
        u64 tx_carrier_errors;
        u64 tx_excess_deferral;
        u64 tx_retry_error;
        u64 rx_frame_error;
        u64 rx_extra_byte;
        u64 rx_late_collision;
        u64 rx_runt;
        u64 rx_frame_too_long;
        u64 rx_over_errors;
        u64 rx_crc_errors;
        u64 rx_frame_align_error;
        u64 rx_length_error;
        u64 rx_unicast;
        u64 rx_multicast;
        u64 rx_broadcast;
        u64 rx_packets; /* should be ifconfig->rx_packets */
        u64 rx_errors_total;
        u64 tx_errors_total;

        /* version 2 stats */
        u64 tx_deferral;
        u64 tx_packets; /* should be ifconfig->tx_packets */
        u64 rx_bytes;   /* should be ifconfig->rx_bytes + 4*rx_packets */
        u64 tx_pause;
        u64 rx_pause;
        u64 rx_drop_frame;

        /* version 3 stats */
        u64 tx_unicast;
        u64 tx_multicast;
        u64 tx_broadcast;
};

#define NV_DEV_STATISTICS_V3_COUNT (sizeof(struct nv_ethtool_stats)/sizeof(u64))
#define NV_DEV_STATISTICS_V2_COUNT (NV_DEV_STATISTICS_V3_COUNT - 3)
#define NV_DEV_STATISTICS_V1_COUNT (NV_DEV_STATISTICS_V2_COUNT - 6)

/* diagnostics */
#define NV_TEST_COUNT_BASE 3
#define NV_TEST_COUNT_EXTENDED 4

static const struct nv_ethtool_str nv_etests_str[] = {
        { "link      (online/offline)" },
        { "register  (offline)       " },
        { "interrupt (offline)       " },
        { "loopback  (offline)       " }
};

struct register_test {
        __u32 reg;
        __u32 mask;
};

static const struct register_test nv_registers_test[] = {
        { NvRegUnknownSetupReg6, 0x01 },
        { NvRegMisc1, 0x03c },
        { NvRegOffloadConfig, 0x03ff },
        { NvRegMulticastAddrA, 0xffffffff },
        { NvRegTxWatermark, 0x0ff },
        { NvRegWakeUpFlags, 0x07777 },
        { 0, 0 }
};

struct nv_skb_map {
        struct sk_buff *skb;
        dma_addr_t dma;
        unsigned int dma_len:31;
        unsigned int dma_single:1;
        struct ring_desc_ex *first_tx_desc;
        struct nv_skb_map *next_tx_ctx;
};

struct nv_txrx_stats {
        u64 stat_rx_packets;
        u64 stat_rx_bytes; /* not always available in HW */
        u64 stat_rx_missed_errors;
        u64 stat_rx_dropped;
        u64 stat_tx_packets; /* not always available in HW */
        u64 stat_tx_bytes;
        u64 stat_tx_dropped;
};

#define nv_txrx_stats_inc(member) \
                __this_cpu_inc(np->txrx_stats->member)
#define nv_txrx_stats_add(member, count) \
                __this_cpu_add(np->txrx_stats->member, (count))

/*
 * SMP locking:
 * All hardware access under netdev_priv(dev)->lock, except the performance
 * critical parts:
 * - rx is (pseudo-) lockless: it relies on the single-threading provided
 *      by the arch code for interrupts.
 * - tx setup is lockless: it relies on netif_tx_lock. Actual submission
 *      needs netdev_priv(dev)->lock :-(
 * - set_multicast_list: preparation lockless, relies on netif_tx_lock.
 *
 * Hardware stats updates are protected by hwstats_lock:
 * - updated by nv_do_stats_poll (timer). This is meant to avoid
 *   integer wraparound in the NIC stats registers, at low frequency
 *   (0.1 Hz)
 * - updated by nv_get_ethtool_stats + nv_get_stats64
 *
 * Software stats are accessed only through 64b synchronization points
 * and are not subject to other synchronization techniques (single
 * update thread on the TX or RX paths).
 */

/* in dev: base, irq */
struct fe_priv {
        spinlock_t lock;

        struct net_device *dev;
        struct napi_struct napi;

        /* hardware stats are updated in syscall and timer */
        spinlock_t hwstats_lock;
        struct nv_ethtool_stats estats;

        int in_shutdown;
        u32 linkspeed;
        int duplex;
        int autoneg;
        int fixed_mode;
        int phyaddr;
        int wolenabled;
        unsigned int phy_oui;
        unsigned int phy_model;
        unsigned int phy_rev;
        u16 gigabit;
        int intr_test;
        int recover_error;
        int quiet_count;

        /* General data: RO fields */
        dma_addr_t ring_addr;
        struct pci_dev *pci_dev;
        u32 orig_mac[2];
        u32 events;
        u32 irqmask;
        u32 desc_ver;
        u32 txrxctl_bits;
        u32 vlanctl_bits;
        u32 driver_data;
        u32 device_id;
        u32 register_size;
        u32 mac_in_use;
        int mgmt_version;
        int mgmt_sema;

        void __iomem *base;

        /* rx specific fields.
         * Locking: Within irq hander or disable_irq+spin_lock(&np->lock);
         */
        union ring_type get_rx, put_rx, last_rx;
        struct nv_skb_map *get_rx_ctx, *put_rx_ctx;
        struct nv_skb_map *last_rx_ctx;
        struct nv_skb_map *rx_skb;

        union ring_type rx_ring;
        unsigned int rx_buf_sz;
        unsigned int pkt_limit;
        struct timer_list oom_kick;
        struct timer_list nic_poll;
        struct timer_list stats_poll;
        u32 nic_poll_irq;
        int rx_ring_size;

        /* RX software stats */
        struct u64_stats_sync swstats_rx_syncp;
        struct nv_txrx_stats __percpu *txrx_stats;

        /* media detection workaround.
         * Locking: Within irq hander or disable_irq+spin_lock(&np->lock);
         */
        int need_linktimer;
        unsigned long link_timeout;
        /*
         * tx specific fields.
         */
        union ring_type get_tx, put_tx, last_tx;
        struct nv_skb_map *get_tx_ctx, *put_tx_ctx;
        struct nv_skb_map *last_tx_ctx;
        struct nv_skb_map *tx_skb;

        union ring_type tx_ring;
        u32 tx_flags;
        int tx_ring_size;
        int tx_limit;
        u32 tx_pkts_in_progress;
        struct nv_skb_map *tx_change_owner;
        struct nv_skb_map *tx_end_flip;
        int tx_stop;

        /* TX software stats */
        struct u64_stats_sync swstats_tx_syncp;

        /* msi/msi-x fields */
        u32 msi_flags;
        struct msix_entry msi_x_entry[NV_MSI_X_MAX_VECTORS];

        /* flow control */
        u32 pause_flags;

        /* power saved state */
        u32 saved_config_space[NV_PCI_REGSZ_MAX/4];

        /* for different msi-x irq type */
        char name_rx[IFNAMSIZ + 3];       /* -rx    */
        char name_tx[IFNAMSIZ + 3];       /* -tx    */
        char name_other[IFNAMSIZ + 6];    /* -other */
};

/*
 * Maximum number of loops until we assume that a bit in the irq mask
 * is stuck. Overridable with module param.
 */
static int max_interrupt_work = 4;

/*
 * Optimization can be either throuput mode or cpu mode
 *
 * Throughput Mode: Every tx and rx packet will generate an interrupt.
 * CPU Mode: Interrupts are controlled by a timer.
 */
enum {
        NV_OPTIMIZATION_MODE_THROUGHPUT,
        NV_OPTIMIZATION_MODE_CPU,
        NV_OPTIMIZATION_MODE_DYNAMIC
};
static int optimization_mode = NV_OPTIMIZATION_MODE_DYNAMIC;

/*
 * Poll interval for timer irq
 *
 * This interval determines how frequent an interrupt is generated.
 * The is value is determined by [(time_in_micro_secs * 100) / (2^10)]
 * Min = 0, and Max = 65535
 */
static int poll_interval = -1;

/*
 * MSI interrupts
 */
enum {
        NV_MSI_INT_DISABLED,
        NV_MSI_INT_ENABLED
};
static int msi = NV_MSI_INT_ENABLED;

/*
 * MSIX interrupts
 */
enum {
        NV_MSIX_INT_DISABLED,
        NV_MSIX_INT_ENABLED
};
static int msix = NV_MSIX_INT_ENABLED;

/*
 * DMA 64bit
 */
enum {
        NV_DMA_64BIT_DISABLED,
        NV_DMA_64BIT_ENABLED
};
static int dma_64bit = NV_DMA_64BIT_ENABLED;

/*
 * Debug output control for tx_timeout
 */
static bool debug_tx_timeout = false;

/*
 * Crossover Detection
 * Realtek 8201 phy + some OEM boards do not work properly.
 */
enum {
        NV_CROSSOVER_DETECTION_DISABLED,
        NV_CROSSOVER_DETECTION_ENABLED
};
static int phy_cross = NV_CROSSOVER_DETECTION_DISABLED;

/*
 * Power down phy when interface is down (persists through reboot;
 * older Linux and other OSes may not power it up again)
 */
static int phy_power_down;

static inline struct fe_priv *get_nvpriv(struct net_device *dev)
{
        return netdev_priv(dev);
}

static inline u8 __iomem *get_hwbase(struct net_device *dev)
{
        return ((struct fe_priv *)netdev_priv(dev))->base;
}

static inline void pci_push(u8 __iomem *base)
{
        /* force out pending posted writes */
        readl(base);
}

static inline u32 nv_descr_getlength(struct ring_desc *prd, u32 v)
{
        return le32_to_cpu(prd->flaglen)
                & ((v == DESC_VER_1) ? LEN_MASK_V1 : LEN_MASK_V2);
}

static inline u32 nv_descr_getlength_ex(struct ring_desc_ex *prd, u32 v)
{
        return le32_to_cpu(prd->flaglen) & LEN_MASK_V2;
}

static bool nv_optimized(struct fe_priv *np)
{
        if (np->desc_ver == DESC_VER_1 || np->desc_ver == DESC_VER_2)
                return false;
        return true;
}

static int reg_delay(struct net_device *dev, int offset, u32 mask, u32 target,
                     int delay, int delaymax)
{
        u8 __iomem *base = get_hwbase(dev);

        pci_push(base);
        do {
                udelay(delay);
                delaymax -= delay;
                if (delaymax < 0)
                        return 1;
        } while ((readl(base + offset) & mask) != target);
        return 0;
}

#define NV_SETUP_RX_RING 0x01
#define NV_SETUP_TX_RING 0x02

static inline u32 dma_low(dma_addr_t addr)
{
        return addr;
}

static inline u32 dma_high(dma_addr_t addr)
{
        return addr>>31>>1;     /* 0 if 32bit, shift down by 32 if 64bit */
}

static void setup_hw_rings(struct net_device *dev, int rxtx_flags)
{
        struct fe_priv *np = get_nvpriv(dev);
        u8 __iomem *base = get_hwbase(dev);

        if (!nv_optimized(np)) {
                if (rxtx_flags & NV_SETUP_RX_RING)
                        writel(dma_low(np->ring_addr), base + NvRegRxRingPhysAddr);
                if (rxtx_flags & NV_SETUP_TX_RING)
                        writel(dma_low(np->ring_addr + np->rx_ring_size*sizeof(struct ring_desc)), base + NvRegTxRingPhysAddr);
        } else {
                if (rxtx_flags & NV_SETUP_RX_RING) {
                        writel(dma_low(np->ring_addr), base + NvRegRxRingPhysAddr);
                        writel(dma_high(np->ring_addr), base + NvRegRxRingPhysAddrHigh);
                }
                if (rxtx_flags & NV_SETUP_TX_RING) {
                        writel(dma_low(np->ring_addr + np->rx_ring_size*sizeof(struct ring_desc_ex)), base + NvRegTxRingPhysAddr);
                        writel(dma_high(np->ring_addr + np->rx_ring_size*sizeof(struct ring_desc_ex)), base + NvRegTxRingPhysAddrHigh);
                }
        }
}

static void free_rings(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);

        if (!nv_optimized(np)) {
                if (np->rx_ring.orig)
                        dma_free_coherent(&np->pci_dev->dev,
                                          sizeof(struct ring_desc) *
                                          (np->rx_ring_size +
                                          np->tx_ring_size),
                                          np->rx_ring.orig, np->ring_addr);
        } else {
                if (np->rx_ring.ex)
                        dma_free_coherent(&np->pci_dev->dev,
                                          sizeof(struct ring_desc_ex) *
                                          (np->rx_ring_size +
                                          np->tx_ring_size),
                                          np->rx_ring.ex, np->ring_addr);
        }
        kfree(np->rx_skb);
        kfree(np->tx_skb);
}

static int using_multi_irqs(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);

        if (!(np->msi_flags & NV_MSI_X_ENABLED) ||
            ((np->msi_flags & NV_MSI_X_VECTORS_MASK) == 0x1))
                return 0;
        else
                return 1;
}

static void nv_txrx_gate(struct net_device *dev, bool gate)
{
        struct fe_priv *np = get_nvpriv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 powerstate;

        if (!np->mac_in_use &&
            (np->driver_data & DEV_HAS_POWER_CNTRL)) {
                powerstate = readl(base + NvRegPowerState2);
                if (gate)
                        powerstate |= NVREG_POWERSTATE2_GATE_CLOCKS;
                else
                        powerstate &= ~NVREG_POWERSTATE2_GATE_CLOCKS;
                writel(powerstate, base + NvRegPowerState2);
        }
}

static void nv_enable_irq(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);

        if (!using_multi_irqs(dev)) {
                if (np->msi_flags & NV_MSI_X_ENABLED)
                        enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_ALL].vector);
                else
                        enable_irq(np->pci_dev->irq);
        } else {
                enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_RX].vector);
                enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_TX].vector);
                enable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_OTHER].vector);
        }
}

static void nv_disable_irq(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);

        if (!using_multi_irqs(dev)) {
                if (np->msi_flags & NV_MSI_X_ENABLED)
                        disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_ALL].vector);
                else
                        disable_irq(np->pci_dev->irq);
        } else {
                disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_RX].vector);
                disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_TX].vector);
                disable_irq(np->msi_x_entry[NV_MSI_X_VECTOR_OTHER].vector);
        }
}

/* In MSIX mode, a write to irqmask behaves as XOR */
static void nv_enable_hw_interrupts(struct net_device *dev, u32 mask)
{
        u8 __iomem *base = get_hwbase(dev);

        writel(mask, base + NvRegIrqMask);
}

static void nv_disable_hw_interrupts(struct net_device *dev, u32 mask)
{
        struct fe_priv *np = get_nvpriv(dev);
        u8 __iomem *base = get_hwbase(dev);

        if (np->msi_flags & NV_MSI_X_ENABLED) {
                writel(mask, base + NvRegIrqMask);
        } else {
                if (np->msi_flags & NV_MSI_ENABLED)
                        writel(0, base + NvRegMSIIrqMask);
                writel(0, base + NvRegIrqMask);
        }
}

static void nv_napi_enable(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);

        napi_enable(&np->napi);
}

static void nv_napi_disable(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);

        napi_disable(&np->napi);
}

#define MII_READ        (-1)
/* mii_rw: read/write a register on the PHY.
 *
 * Caller must guarantee serialization
 */
static int mii_rw(struct net_device *dev, int addr, int miireg, int value)
{
        u8 __iomem *base = get_hwbase(dev);
        u32 reg;
        int retval;

        writel(NVREG_MIISTAT_MASK_RW, base + NvRegMIIStatus);

        reg = readl(base + NvRegMIIControl);
        if (reg & NVREG_MIICTL_INUSE) {
                writel(NVREG_MIICTL_INUSE, base + NvRegMIIControl);
                udelay(NV_MIIBUSY_DELAY);
        }

        reg = (addr << NVREG_MIICTL_ADDRSHIFT) | miireg;
        if (value != MII_READ) {
                writel(value, base + NvRegMIIData);
                reg |= NVREG_MIICTL_WRITE;
        }
        writel(reg, base + NvRegMIIControl);

        if (reg_delay(dev, NvRegMIIControl, NVREG_MIICTL_INUSE, 0,
                        NV_MIIPHY_DELAY, NV_MIIPHY_DELAYMAX)) {
                retval = -1;
        } else if (value != MII_READ) {
                /* it was a write operation - fewer failures are detectable */
                retval = 0;
        } else if (readl(base + NvRegMIIStatus) & NVREG_MIISTAT_ERROR) {
                retval = -1;
        } else {
                retval = readl(base + NvRegMIIData);
        }

        return retval;
}

static int phy_reset(struct net_device *dev, u32 bmcr_setup)
{
        struct fe_priv *np = netdev_priv(dev);
        u32 miicontrol;
        unsigned int tries = 0;

        miicontrol = BMCR_RESET | bmcr_setup;
        if (mii_rw(dev, np->phyaddr, MII_BMCR, miicontrol))
                return -1;

        /* wait for 500ms */
        msleep(500);

        /* must wait till reset is deasserted */
        while (miicontrol & BMCR_RESET) {
                usleep_range(10000, 20000);
                miicontrol = mii_rw(dev, np->phyaddr, MII_BMCR, MII_READ);
                /* FIXME: 100 tries seem excessive */
                if (tries++ > 100)
                        return -1;
        }
        return 0;
}

static int init_realtek_8211b(struct net_device *dev, struct fe_priv *np)
{
        static const struct {
                int reg;
                int init;
        } ri[] = {
                { PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT1 },
                { PHY_REALTEK_INIT_REG2, PHY_REALTEK_INIT2 },
                { PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT3 },
                { PHY_REALTEK_INIT_REG3, PHY_REALTEK_INIT4 },
                { PHY_REALTEK_INIT_REG4, PHY_REALTEK_INIT5 },
                { PHY_REALTEK_INIT_REG5, PHY_REALTEK_INIT6 },
                { PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT1 },
        };
        int i;

        for (i = 0; i < ARRAY_SIZE(ri); i++) {
                if (mii_rw(dev, np->phyaddr, ri[i].reg, ri[i].init))
                        return PHY_ERROR;
        }

        return 0;
}

static int init_realtek_8211c(struct net_device *dev, struct fe_priv *np)
{
        u32 reg;
        u8 __iomem *base = get_hwbase(dev);
        u32 powerstate = readl(base + NvRegPowerState2);

        /* need to perform hw phy reset */
        powerstate |= NVREG_POWERSTATE2_PHY_RESET;
        writel(powerstate, base + NvRegPowerState2);
        msleep(25);

        powerstate &= ~NVREG_POWERSTATE2_PHY_RESET;
        writel(powerstate, base + NvRegPowerState2);
        msleep(25);

        reg = mii_rw(dev, np->phyaddr, PHY_REALTEK_INIT_REG6, MII_READ);
        reg |= PHY_REALTEK_INIT9;
        if (mii_rw(dev, np->phyaddr, PHY_REALTEK_INIT_REG6, reg))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT10))
                return PHY_ERROR;
        reg = mii_rw(dev, np->phyaddr, PHY_REALTEK_INIT_REG7, MII_READ);
        if (!(reg & PHY_REALTEK_INIT11)) {
                reg |= PHY_REALTEK_INIT11;
                if (mii_rw(dev, np->phyaddr, PHY_REALTEK_INIT_REG7, reg))
                        return PHY_ERROR;
        }
        if (mii_rw(dev, np->phyaddr,
                   PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT1))
                return PHY_ERROR;

        return 0;
}

static int init_realtek_8201(struct net_device *dev, struct fe_priv *np)
{
        u32 phy_reserved;

        if (np->driver_data & DEV_NEED_PHY_INIT_FIX) {
                phy_reserved = mii_rw(dev, np->phyaddr,
                                      PHY_REALTEK_INIT_REG6, MII_READ);
                phy_reserved |= PHY_REALTEK_INIT7;
                if (mii_rw(dev, np->phyaddr,
                           PHY_REALTEK_INIT_REG6, phy_reserved))
                        return PHY_ERROR;
        }

        return 0;
}

static int init_realtek_8201_cross(struct net_device *dev, struct fe_priv *np)
{
        u32 phy_reserved;

        if (phy_cross == NV_CROSSOVER_DETECTION_DISABLED) {
                if (mii_rw(dev, np->phyaddr,
                           PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT3))
                        return PHY_ERROR;
                phy_reserved = mii_rw(dev, np->phyaddr,
                                      PHY_REALTEK_INIT_REG2, MII_READ);
                phy_reserved &= ~PHY_REALTEK_INIT_MSK1;
                phy_reserved |= PHY_REALTEK_INIT3;
                if (mii_rw(dev, np->phyaddr,
                           PHY_REALTEK_INIT_REG2, phy_reserved))
                        return PHY_ERROR;
                if (mii_rw(dev, np->phyaddr,
                           PHY_REALTEK_INIT_REG1, PHY_REALTEK_INIT1))
                        return PHY_ERROR;
        }

        return 0;
}

static int init_cicada(struct net_device *dev, struct fe_priv *np,
                       u32 phyinterface)
{
        u32 phy_reserved;

        if (phyinterface & PHY_RGMII) {
                phy_reserved = mii_rw(dev, np->phyaddr, MII_RESV1, MII_READ);
                phy_reserved &= ~(PHY_CICADA_INIT1 | PHY_CICADA_INIT2);
                phy_reserved |= (PHY_CICADA_INIT3 | PHY_CICADA_INIT4);
                if (mii_rw(dev, np->phyaddr, MII_RESV1, phy_reserved))
                        return PHY_ERROR;
                phy_reserved = mii_rw(dev, np->phyaddr, MII_NCONFIG, MII_READ);
                phy_reserved |= PHY_CICADA_INIT5;
                if (mii_rw(dev, np->phyaddr, MII_NCONFIG, phy_reserved))
                        return PHY_ERROR;
        }
        phy_reserved = mii_rw(dev, np->phyaddr, MII_SREVISION, MII_READ);
        phy_reserved |= PHY_CICADA_INIT6;
        if (mii_rw(dev, np->phyaddr, MII_SREVISION, phy_reserved))
                return PHY_ERROR;

        return 0;
}

static int init_vitesse(struct net_device *dev, struct fe_priv *np)
{
        u32 phy_reserved;

        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG1, PHY_VITESSE_INIT1))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG2, PHY_VITESSE_INIT2))
                return PHY_ERROR;
        phy_reserved = mii_rw(dev, np->phyaddr,
                              PHY_VITESSE_INIT_REG4, MII_READ);
        if (mii_rw(dev, np->phyaddr, PHY_VITESSE_INIT_REG4, phy_reserved))
                return PHY_ERROR;
        phy_reserved = mii_rw(dev, np->phyaddr,
                              PHY_VITESSE_INIT_REG3, MII_READ);
        phy_reserved &= ~PHY_VITESSE_INIT_MSK1;
        phy_reserved |= PHY_VITESSE_INIT3;
        if (mii_rw(dev, np->phyaddr, PHY_VITESSE_INIT_REG3, phy_reserved))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG2, PHY_VITESSE_INIT4))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG2, PHY_VITESSE_INIT5))
                return PHY_ERROR;
        phy_reserved = mii_rw(dev, np->phyaddr,
                              PHY_VITESSE_INIT_REG4, MII_READ);
        phy_reserved &= ~PHY_VITESSE_INIT_MSK1;
        phy_reserved |= PHY_VITESSE_INIT3;
        if (mii_rw(dev, np->phyaddr, PHY_VITESSE_INIT_REG4, phy_reserved))
                return PHY_ERROR;
        phy_reserved = mii_rw(dev, np->phyaddr,
                              PHY_VITESSE_INIT_REG3, MII_READ);
        if (mii_rw(dev, np->phyaddr, PHY_VITESSE_INIT_REG3, phy_reserved))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG2, PHY_VITESSE_INIT6))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG2, PHY_VITESSE_INIT7))
                return PHY_ERROR;
        phy_reserved = mii_rw(dev, np->phyaddr,
                              PHY_VITESSE_INIT_REG4, MII_READ);
        if (mii_rw(dev, np->phyaddr, PHY_VITESSE_INIT_REG4, phy_reserved))
                return PHY_ERROR;
        phy_reserved = mii_rw(dev, np->phyaddr,
                              PHY_VITESSE_INIT_REG3, MII_READ);
        phy_reserved &= ~PHY_VITESSE_INIT_MSK2;
        phy_reserved |= PHY_VITESSE_INIT8;
        if (mii_rw(dev, np->phyaddr, PHY_VITESSE_INIT_REG3, phy_reserved))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG2, PHY_VITESSE_INIT9))
                return PHY_ERROR;
        if (mii_rw(dev, np->phyaddr,
                   PHY_VITESSE_INIT_REG1, PHY_VITESSE_INIT10))
                return PHY_ERROR;

        return 0;
}

static int phy_init(struct net_device *dev)
{
        struct fe_priv *np = get_nvpriv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 phyinterface;
        u32 mii_status, mii_control, mii_control_1000, reg;

        /* phy errata for E3016 phy */
        if (np->phy_model == PHY_MODEL_MARVELL_E3016) {
                reg = mii_rw(dev, np->phyaddr, MII_NCONFIG, MII_READ);
                reg &= ~PHY_MARVELL_E3016_INITMASK;
                if (mii_rw(dev, np->phyaddr, MII_NCONFIG, reg)) {
                        netdev_info(dev, "%s: phy write to errata reg failed\n",
                                    pci_name(np->pci_dev));
                        return PHY_ERROR;
                }
        }
        if (np->phy_oui == PHY_OUI_REALTEK) {
                if (np->phy_model == PHY_MODEL_REALTEK_8211 &&
                    np->phy_rev == PHY_REV_REALTEK_8211B) {
                        if (init_realtek_8211b(dev, np)) {
                                netdev_info(dev, "%s: phy init failed\n",
                                            pci_name(np->pci_dev));
                                return PHY_ERROR;
                        }
                } else if (np->phy_model == PHY_MODEL_REALTEK_8211 &&
                           np->phy_rev == PHY_REV_REALTEK_8211C) {
                        if (init_realtek_8211c(dev, np)) {
                                netdev_info(dev, "%s: phy init failed\n",
                                            pci_name(np->pci_dev));
                                return PHY_ERROR;
                        }
                } else if (np->phy_model == PHY_MODEL_REALTEK_8201) {
                        if (init_realtek_8201(dev, np)) {
                                netdev_info(dev, "%s: phy init failed\n",
                                            pci_name(np->pci_dev));
                                return PHY_ERROR;
                        }
                }
        }

        /* set advertise register */
        reg = mii_rw(dev, np->phyaddr, MII_ADVERTISE, MII_READ);
        reg |= (ADVERTISE_10HALF | ADVERTISE_10FULL |
                ADVERTISE_100HALF | ADVERTISE_100FULL |
                ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP);
        if (mii_rw(dev, np->phyaddr, MII_ADVERTISE, reg)) {
                netdev_info(dev, "%s: phy write to advertise failed\n",
                            pci_name(np->pci_dev));
                return PHY_ERROR;
        }

        /* get phy interface type */
        phyinterface = readl(base + NvRegPhyInterface);

        /* see if gigabit phy */
        mii_status = mii_rw(dev, np->phyaddr, MII_BMSR, MII_READ);
        if (mii_status & PHY_GIGABIT) {
                np->gigabit = PHY_GIGABIT;
                mii_control_1000 = mii_rw(dev, np->phyaddr,
                                          MII_CTRL1000, MII_READ);
                mii_control_1000 &= ~ADVERTISE_1000HALF;
                if (phyinterface & PHY_RGMII)
                        mii_control_1000 |= ADVERTISE_1000FULL;
                else
                        mii_control_1000 &= ~ADVERTISE_1000FULL;

                if (mii_rw(dev, np->phyaddr, MII_CTRL1000, mii_control_1000)) {
                        netdev_info(dev, "%s: phy init failed\n",
                                    pci_name(np->pci_dev));
                        return PHY_ERROR;
                }
        } else
                np->gigabit = 0;

        mii_control = mii_rw(dev, np->phyaddr, MII_BMCR, MII_READ);
        mii_control |= BMCR_ANENABLE;

        if (np->phy_oui == PHY_OUI_REALTEK &&
            np->phy_model == PHY_MODEL_REALTEK_8211 &&
            np->phy_rev == PHY_REV_REALTEK_8211C) {
                /* start autoneg since we already performed hw reset above */
                mii_control |= BMCR_ANRESTART;
                if (mii_rw(dev, np->phyaddr, MII_BMCR, mii_control)) {
                        netdev_info(dev, "%s: phy init failed\n",
                                    pci_name(np->pci_dev));
                        return PHY_ERROR;
                }
        } else {
                /* reset the phy
                 * (certain phys need bmcr to be setup with reset)
                 */
                if (phy_reset(dev, mii_control)) {
                        netdev_info(dev, "%s: phy reset failed\n",
                                    pci_name(np->pci_dev));
                        return PHY_ERROR;
                }
        }

        /* phy vendor specific configuration */
        if (np->phy_oui == PHY_OUI_CICADA) {
                if (init_cicada(dev, np, phyinterface)) {
                        netdev_info(dev, "%s: phy init failed\n",
                                    pci_name(np->pci_dev));
                        return PHY_ERROR;
                }
        } else if (np->phy_oui == PHY_OUI_VITESSE) {
                if (init_vitesse(dev, np)) {
                        netdev_info(dev, "%s: phy init failed\n",
                                    pci_name(np->pci_dev));
                        return PHY_ERROR;
                }
        } else if (np->phy_oui == PHY_OUI_REALTEK) {
                if (np->phy_model == PHY_MODEL_REALTEK_8211 &&
                    np->phy_rev == PHY_REV_REALTEK_8211B) {
                        /* reset could have cleared these out, set them back */
                        if (init_realtek_8211b(dev, np)) {
                                netdev_info(dev, "%s: phy init failed\n",
                                            pci_name(np->pci_dev));
                                return PHY_ERROR;
                        }
                } else if (np->phy_model == PHY_MODEL_REALTEK_8201) {
                        if (init_realtek_8201(dev, np) ||
                            init_realtek_8201_cross(dev, np)) {
                                netdev_info(dev, "%s: phy init failed\n",
                                            pci_name(np->pci_dev));
                                return PHY_ERROR;
                        }
                }
        }

        /* some phys clear out pause advertisement on reset, set it back */
        mii_rw(dev, np->phyaddr, MII_ADVERTISE, reg);

        /* restart auto negotiation, power down phy */
        mii_control = mii_rw(dev, np->phyaddr, MII_BMCR, MII_READ);
        mii_control |= (BMCR_ANRESTART | BMCR_ANENABLE);
        if (phy_power_down)
                mii_control |= BMCR_PDOWN;
        if (mii_rw(dev, np->phyaddr, MII_BMCR, mii_control))
                return PHY_ERROR;

        return 0;
}

static void nv_start_rx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 rx_ctrl = readl(base + NvRegReceiverControl);

        /* Already running? Stop it. */
        if ((readl(base + NvRegReceiverControl) & NVREG_RCVCTL_START) && !np->mac_in_use) {
                rx_ctrl &= ~NVREG_RCVCTL_START;
                writel(rx_ctrl, base + NvRegReceiverControl);
                pci_push(base);
        }
        writel(np->linkspeed, base + NvRegLinkSpeed);
        pci_push(base);
        rx_ctrl |= NVREG_RCVCTL_START;
        if (np->mac_in_use)
                rx_ctrl &= ~NVREG_RCVCTL_RX_PATH_EN;
        writel(rx_ctrl, base + NvRegReceiverControl);
        pci_push(base);
}

static void nv_stop_rx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 rx_ctrl = readl(base + NvRegReceiverControl);

        if (!np->mac_in_use)
                rx_ctrl &= ~NVREG_RCVCTL_START;
        else
                rx_ctrl |= NVREG_RCVCTL_RX_PATH_EN;
        writel(rx_ctrl, base + NvRegReceiverControl);
        if (reg_delay(dev, NvRegReceiverStatus, NVREG_RCVSTAT_BUSY, 0,
                      NV_RXSTOP_DELAY1, NV_RXSTOP_DELAY1MAX))
                netdev_info(dev, "%s: ReceiverStatus remained busy\n",
                            __func__);

        udelay(NV_RXSTOP_DELAY2);
        if (!np->mac_in_use)
                writel(0, base + NvRegLinkSpeed);
}

static void nv_start_tx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 tx_ctrl = readl(base + NvRegTransmitterControl);

        tx_ctrl |= NVREG_XMITCTL_START;
        if (np->mac_in_use)
                tx_ctrl &= ~NVREG_XMITCTL_TX_PATH_EN;
        writel(tx_ctrl, base + NvRegTransmitterControl);
        pci_push(base);
}

static void nv_stop_tx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 tx_ctrl = readl(base + NvRegTransmitterControl);

        if (!np->mac_in_use)
                tx_ctrl &= ~NVREG_XMITCTL_START;
        else
                tx_ctrl |= NVREG_XMITCTL_TX_PATH_EN;
        writel(tx_ctrl, base + NvRegTransmitterControl);
        if (reg_delay(dev, NvRegTransmitterStatus, NVREG_XMITSTAT_BUSY, 0,
                      NV_TXSTOP_DELAY1, NV_TXSTOP_DELAY1MAX))
                netdev_info(dev, "%s: TransmitterStatus remained busy\n",
                            __func__);

        udelay(NV_TXSTOP_DELAY2);
        if (!np->mac_in_use)
                writel(readl(base + NvRegTransmitPoll) & NVREG_TRANSMITPOLL_MAC_ADDR_REV,
                       base + NvRegTransmitPoll);
}

static void nv_start_rxtx(struct net_device *dev)
{
        nv_start_rx(dev);
        nv_start_tx(dev);
}

static void nv_stop_rxtx(struct net_device *dev)
{
        nv_stop_rx(dev);
        nv_stop_tx(dev);
}

static void nv_txrx_reset(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);

        writel(NVREG_TXRXCTL_BIT2 | NVREG_TXRXCTL_RESET | np->txrxctl_bits, base + NvRegTxRxControl);
        pci_push(base);
        udelay(NV_TXRX_RESET_DELAY);
        writel(NVREG_TXRXCTL_BIT2 | np->txrxctl_bits, base + NvRegTxRxControl);
        pci_push(base);
}

static void nv_mac_reset(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);
        u32 temp1, temp2, temp3;

        writel(NVREG_TXRXCTL_BIT2 | NVREG_TXRXCTL_RESET | np->txrxctl_bits, base + NvRegTxRxControl);
        pci_push(base);

        /* save registers since they will be cleared on reset */
        temp1 = readl(base + NvRegMacAddrA);
        temp2 = readl(base + NvRegMacAddrB);
        temp3 = readl(base + NvRegTransmitPoll);

        writel(NVREG_MAC_RESET_ASSERT, base + NvRegMacReset);
        pci_push(base);
        udelay(NV_MAC_RESET_DELAY);
        writel(0, base + NvRegMacReset);
        pci_push(base);
        udelay(NV_MAC_RESET_DELAY);

        /* restore saved registers */
        writel(temp1, base + NvRegMacAddrA);
        writel(temp2, base + NvRegMacAddrB);
        writel(temp3, base + NvRegTransmitPoll);

        writel(NVREG_TXRXCTL_BIT2 | np->txrxctl_bits, base + NvRegTxRxControl);
        pci_push(base);
}

/* Caller must appropriately lock netdev_priv(dev)->hwstats_lock */
static void nv_update_stats(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        u8 __iomem *base = get_hwbase(dev);

        lockdep_assert_held(&np->hwstats_lock);

        /* query hardware */
        np->estats.tx_bytes += readl(base + NvRegTxCnt);
        np->estats.tx_zero_rexmt += readl(base + NvRegTxZeroReXmt);
        np->estats.tx_one_rexmt += readl(base + NvRegTxOneReXmt);
        np->estats.tx_many_rexmt += readl(base + NvRegTxManyReXmt);
        np->estats.tx_late_collision += readl(base + NvRegTxLateCol);
        np->estats.tx_fifo_errors += readl(base + NvRegTxUnderflow);
        np->estats.tx_carrier_errors += readl(base + NvRegTxLossCarrier);
        np->estats.tx_excess_deferral += readl(base + NvRegTxExcessDef);
        np->estats.tx_retry_error += readl(base + NvRegTxRetryErr);
        np->estats.rx_frame_error += readl(base + NvRegRxFrameErr);
        np->estats.rx_extra_byte += readl(base + NvRegRxExtraByte);
        np->estats.rx_late_collision += readl(base + NvRegRxLateCol);
        np->estats.rx_runt += readl(base + NvRegRxRunt);
        np->estats.rx_frame_too_long += readl(base + NvRegRxFrameTooLong);
        np->estats.rx_over_errors += readl(base + NvRegRxOverflow);
        np->estats.rx_crc_errors += readl(base + NvRegRxFCSErr);
        np->estats.rx_frame_align_error += readl(base + NvRegRxFrameAlignErr);
        np->estats.rx_length_error += readl(base + NvRegRxLenErr);
        np->estats.rx_unicast += readl(base + NvRegRxUnicast);
        np->estats.rx_multicast += readl(base + NvRegRxMulticast);
        np->estats.rx_broadcast += readl(base + NvRegRxBroadcast);
        np->estats.rx_packets =
                np->estats.rx_unicast +
                np->estats.rx_multicast +
                np->estats.rx_broadcast;
        np->estats.rx_errors_total =
                np->estats.rx_crc_errors +
                np->estats.rx_over_errors +
                np->estats.rx_frame_error +
                (np->estats.rx_frame_align_error - np->estats.rx_extra_byte) +
                np->estats.rx_late_collision +
                np->estats.rx_runt +
                np->estats.rx_frame_too_long;
        np->estats.tx_errors_total =
                np->estats.tx_late_collision +
                np->estats.tx_fifo_errors +
                np->estats.tx_carrier_errors +
                np->estats.tx_excess_deferral +
                np->estats.tx_retry_error;

        if (np->driver_data & DEV_HAS_STATISTICS_V2) {
                np->estats.tx_deferral += readl(base + NvRegTxDef);
                np->estats.tx_packets += readl(base + NvRegTxFrame);
                np->estats.rx_bytes += readl(base + NvRegRxCnt);
                np->estats.tx_pause += readl(base + NvRegTxPause);
                np->estats.rx_pause += readl(base + NvRegRxPause);
                np->estats.rx_drop_frame += readl(base + NvRegRxDropFrame);
                np->estats.rx_errors_total += np->estats.rx_drop_frame;
        }

        if (np->driver_data & DEV_HAS_STATISTICS_V3) {
                np->estats.tx_unicast += readl(base + NvRegTxUnicast);
                np->estats.tx_multicast += readl(base + NvRegTxMulticast);
                np->estats.tx_broadcast += readl(base + NvRegTxBroadcast);
        }
}

static void nv_get_stats(int cpu, struct fe_priv *np,
                         struct rtnl_link_stats64 *storage)
{
        struct nv_txrx_stats *src = per_cpu_ptr(np->txrx_stats, cpu);
        unsigned int syncp_start;
        u64 rx_packets, rx_bytes, rx_dropped, rx_missed_errors;
        u64 tx_packets, tx_bytes, tx_dropped;

        do {
                syncp_start = u64_stats_fetch_begin_irq(&np->swstats_rx_syncp);
                rx_packets       = src->stat_rx_packets;
                rx_bytes         = src->stat_rx_bytes;
                rx_dropped       = src->stat_rx_dropped;
                rx_missed_errors = src->stat_rx_missed_errors;
        } while (u64_stats_fetch_retry_irq(&np->swstats_rx_syncp, syncp_start));

        storage->rx_packets       += rx_packets;
        storage->rx_bytes         += rx_bytes;
        storage->rx_dropped       += rx_dropped;
        storage->rx_missed_errors += rx_missed_errors;

        do {
                syncp_start = u64_stats_fetch_begin_irq(&np->swstats_tx_syncp);
                tx_packets  = src->stat_tx_packets;
                tx_bytes    = src->stat_tx_bytes;
                tx_dropped  = src->stat_tx_dropped;
        } while (u64_stats_fetch_retry_irq(&np->swstats_tx_syncp, syncp_start));

        storage->tx_packets += tx_packets;
        storage->tx_bytes   += tx_bytes;
        storage->tx_dropped += tx_dropped;
}

/*
 * nv_get_stats64: dev->ndo_get_stats64 function
 * Get latest stats value from the nic.
 * Called with read_lock(&dev_base_lock) held for read -
 * only synchronized against unregister_netdevice.
 */
static void
nv_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *storage)
        __acquires(&netdev_priv(dev)->hwstats_lock)
        __releases(&netdev_priv(dev)->hwstats_lock)
{
        struct fe_priv *np = netdev_priv(dev);
        int cpu;

        /*
         * Note: because HW stats are not always available and for
         * consistency reasons, the following ifconfig stats are
         * managed by software: rx_bytes, tx_bytes, rx_packets and
         * tx_packets. The related hardware stats reported by ethtool
         * should be equivalent to these ifconfig stats, with 4
         * additional bytes per packet (Ethernet FCS CRC), except for
         * tx_packets when TSO kicks in.
         */

        /* software stats */
        for_each_online_cpu(cpu)
                nv_get_stats(cpu, np, storage);

        /* If the nic supports hw counters then retrieve latest values */
        if (np->driver_data & DEV_HAS_STATISTICS_V123) {
                spin_lock_bh(&np->hwstats_lock);

                nv_update_stats(dev);

                /* generic stats */
                storage->rx_errors = np->estats.rx_errors_total;
                storage->tx_errors = np->estats.tx_errors_total;

                /* meaningful only when NIC supports stats v3 */
                storage->multicast = np->estats.rx_multicast;

                /* detailed rx_errors */
                storage->rx_length_errors = np->estats.rx_length_error;
                storage->rx_over_errors   = np->estats.rx_over_errors;
                storage->rx_crc_errors    = np->estats.rx_crc_errors;
                storage->rx_frame_errors  = np->estats.rx_frame_align_error;
                storage->rx_fifo_errors   = np->estats.rx_drop_frame;

                /* detailed tx_errors */
                storage->tx_carrier_errors = np->estats.tx_carrier_errors;
                storage->tx_fifo_errors    = np->estats.tx_fifo_errors;

                spin_unlock_bh(&np->hwstats_lock);
        }
}

/*
 * nv_alloc_rx: fill rx ring entries.
 * Return 1 if the allocations for the skbs failed and the
 * rx engine is without Available descriptors
 */
static int nv_alloc_rx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        struct ring_desc *less_rx;

        less_rx = np->get_rx.orig;
        if (less_rx-- == np->rx_ring.orig)
                less_rx = np->last_rx.orig;

        while (np->put_rx.orig != less_rx) {
                struct sk_buff *skb = netdev_alloc_skb(dev, np->rx_buf_sz + NV_RX_ALLOC_PAD);
                if (likely(skb)) {
                        np->put_rx_ctx->skb = skb;
                        np->put_rx_ctx->dma = dma_map_single(&np->pci_dev->dev,
                                                             skb->data,
                                                             skb_tailroom(skb),
                                                             DMA_FROM_DEVICE);
                        if (unlikely(dma_mapping_error(&np->pci_dev->dev,
                                                       np->put_rx_ctx->dma))) {
                                kfree_skb(skb);
                                goto packet_dropped;
                        }
                        np->put_rx_ctx->dma_len = skb_tailroom(skb);
                        np->put_rx.orig->buf = cpu_to_le32(np->put_rx_ctx->dma);
                        wmb();
                        np->put_rx.orig->flaglen = cpu_to_le32(np->rx_buf_sz | NV_RX_AVAIL);
                        if (unlikely(np->put_rx.orig++ == np->last_rx.orig))
                                np->put_rx.orig = np->rx_ring.orig;
                        if (unlikely(np->put_rx_ctx++ == np->last_rx_ctx))
                                np->put_rx_ctx = np->rx_skb;
                } else {
packet_dropped:
                        u64_stats_update_begin(&np->swstats_rx_syncp);
                        nv_txrx_stats_inc(stat_rx_dropped);
                        u64_stats_update_end(&np->swstats_rx_syncp);
                        return 1;
                }
        }
        return 0;
}

static int nv_alloc_rx_optimized(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        struct ring_desc_ex *less_rx;

        less_rx = np->get_rx.ex;
        if (less_rx-- == np->rx_ring.ex)
                less_rx = np->last_rx.ex;

        while (np->put_rx.ex != less_rx) {
                struct sk_buff *skb = netdev_alloc_skb(dev, np->rx_buf_sz + NV_RX_ALLOC_PAD);
                if (likely(skb)) {
                        np->put_rx_ctx->skb = skb;
                        np->put_rx_ctx->dma = dma_map_single(&np->pci_dev->dev,
                                                             skb->data,
                                                             skb_tailroom(skb),
                                                             DMA_FROM_DEVICE);
                        if (unlikely(dma_mapping_error(&np->pci_dev->dev,
                                                       np->put_rx_ctx->dma))) {
                                kfree_skb(skb);
                                goto packet_dropped;
                        }
                        np->put_rx_ctx->dma_len = skb_tailroom(skb);
                        np->put_rx.ex->bufhigh = cpu_to_le32(dma_high(np->put_rx_ctx->dma));
                        np->put_rx.ex->buflow = cpu_to_le32(dma_low(np->put_rx_ctx->dma));
                        wmb();
                        np->put_rx.ex->flaglen = cpu_to_le32(np->rx_buf_sz | NV_RX2_AVAIL);
                        if (unlikely(np->put_rx.ex++ == np->last_rx.ex))
                                np->put_rx.ex = np->rx_ring.ex;
                        if (unlikely(np->put_rx_ctx++ == np->last_rx_ctx))
                                np->put_rx_ctx = np->rx_skb;
                } else {
packet_dropped:
                        u64_stats_update_begin(&np->swstats_rx_syncp);
                        nv_txrx_stats_inc(stat_rx_dropped);
                        u64_stats_update_end(&np->swstats_rx_syncp);
                        return 1;
                }
        }
        return 0;
}

/* If rx bufs are exhausted called after 50ms to attempt to refresh */
static void nv_do_rx_refill(struct timer_list *t)
{
        struct fe_priv *np = from_timer(np, t, oom_kick);

        /* Just reschedule NAPI rx processing */
        napi_schedule(&np->napi);
}

static void nv_init_rx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        int i;

        np->get_rx = np->rx_ring;
        np->put_rx = np->rx_ring;

        if (!nv_optimized(np))
                np->last_rx.orig = &np->rx_ring.orig[np->rx_ring_size-1];
        else
                np->last_rx.ex = &np->rx_ring.ex[np->rx_ring_size-1];
        np->get_rx_ctx = np->rx_skb;
        np->put_rx_ctx = np->rx_skb;
        np->last_rx_ctx = &np->rx_skb[np->rx_ring_size-1];

        for (i = 0; i < np->rx_ring_size; i++) {
                if (!nv_optimized(np)) {
                        np->rx_ring.orig[i].flaglen = 0;
                        np->rx_ring.orig[i].buf = 0;
                } else {
                        np->rx_ring.ex[i].flaglen = 0;
                        np->rx_ring.ex[i].txvlan = 0;
                        np->rx_ring.ex[i].bufhigh = 0;
                        np->rx_ring.ex[i].buflow = 0;
                }
                np->rx_skb[i].skb = NULL;
                np->rx_skb[i].dma = 0;
        }
}

static void nv_init_tx(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);
        int i;

        np->get_tx = np->tx_ring;
        np->put_tx = np->tx_ring;

        if (!nv_optimized(np))
                np->last_tx.orig = &np->tx_ring.orig[np->tx_ring_size-1];
        else
                np->last_tx.ex = &np->tx_ring.ex[np->tx_ring_size-1];
        np->get_tx_ctx = np->tx_skb;
        np->put_tx_ctx = np->tx_skb;
        np->last_tx_ctx = &np->tx_skb[np->tx_ring_size-1];
        netdev_reset_queue(np->dev);
        np->tx_pkts_in_progress = 0;
        np->tx_change_owner = NULL;
        np->tx_end_flip = NULL;
        np->tx_stop = 0;

        for (i = 0; i < np->tx_ring_size; i++) {
                if (!nv_optimized(np)) {
                        np->tx_ring.orig[i].flaglen = 0;
                        np->tx_ring.orig[i].buf = 0;
                } else {
                        np->tx_ring.ex[i].flaglen = 0;
                        np->tx_ring.ex[i].txvlan = 0;
                        np->tx_ring.ex[i].bufhigh = 0;
                        np->tx_ring.ex[i].buflow = 0;
                }
                np->tx_skb[i].skb = NULL;
                np->tx_skb[i].dma = 0;
                np->tx_skb[i].dma_len = 0;
                np->tx_skb[i].dma_single = 0;
                np->tx_skb[i].first_tx_desc = NULL;
                np->tx_skb[i].next_tx_ctx = NULL;
        }
}

static int nv_init_ring(struct net_device *dev)
{
        struct fe_priv *np = netdev_priv(dev);

        nv_init_tx(dev);
        nv_init_rx(dev);

        if (!nv_optimized(np))
                return nv_alloc_rx(dev);
        else
                return nv_alloc_rx_optimized(dev);
}

static void nv_unmap_txskb(struct fe_priv *np, struct nv_skb_map *tx_skb)
{
        if (tx_skb->dma) {