linux/drivers/net/ethernet/broadcom/bcmsysport.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Broadcom BCM7xxx System Port Ethernet MAC driver
   4 *
   5 * Copyright (C) 2014 Broadcom Corporation
   6 */
   7
   8#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
   9
  10#include <linux/init.h>
  11#include <linux/interrupt.h>
  12#include <linux/module.h>
  13#include <linux/kernel.h>
  14#include <linux/netdevice.h>
  15#include <linux/dsa/brcm.h>
  16#include <linux/etherdevice.h>
  17#include <linux/platform_device.h>
  18#include <linux/of.h>
  19#include <linux/of_net.h>
  20#include <linux/of_mdio.h>
  21#include <linux/phy.h>
  22#include <linux/phy_fixed.h>
  23#include <net/dsa.h>
  24#include <linux/clk.h>
  25#include <net/ip.h>
  26#include <net/ipv6.h>
  27
  28#include "bcmsysport.h"
  29
  30/* I/O accessors register helpers */
  31#define BCM_SYSPORT_IO_MACRO(name, offset) \
  32static inline u32 name##_readl(struct bcm_sysport_priv *priv, u32 off)  \
  33{                                                                       \
  34        u32 reg = readl_relaxed(priv->base + offset + off);             \
  35        return reg;                                                     \
  36}                                                                       \
  37static inline void name##_writel(struct bcm_sysport_priv *priv,         \
  38                                  u32 val, u32 off)                     \
  39{                                                                       \
  40        writel_relaxed(val, priv->base + offset + off);                 \
  41}                                                                       \
  42
  43BCM_SYSPORT_IO_MACRO(intrl2_0, SYS_PORT_INTRL2_0_OFFSET);
  44BCM_SYSPORT_IO_MACRO(intrl2_1, SYS_PORT_INTRL2_1_OFFSET);
  45BCM_SYSPORT_IO_MACRO(umac, SYS_PORT_UMAC_OFFSET);
  46BCM_SYSPORT_IO_MACRO(gib, SYS_PORT_GIB_OFFSET);
  47BCM_SYSPORT_IO_MACRO(tdma, SYS_PORT_TDMA_OFFSET);
  48BCM_SYSPORT_IO_MACRO(rxchk, SYS_PORT_RXCHK_OFFSET);
  49BCM_SYSPORT_IO_MACRO(txchk, SYS_PORT_TXCHK_OFFSET);
  50BCM_SYSPORT_IO_MACRO(rbuf, SYS_PORT_RBUF_OFFSET);
  51BCM_SYSPORT_IO_MACRO(tbuf, SYS_PORT_TBUF_OFFSET);
  52BCM_SYSPORT_IO_MACRO(topctrl, SYS_PORT_TOPCTRL_OFFSET);
  53
  54/* On SYSTEMPORT Lite, any register after RDMA_STATUS has the exact
  55 * same layout, except it has been moved by 4 bytes up, *sigh*
  56 */
  57static inline u32 rdma_readl(struct bcm_sysport_priv *priv, u32 off)
  58{
  59        if (priv->is_lite && off >= RDMA_STATUS)
  60                off += 4;
  61        return readl_relaxed(priv->base + SYS_PORT_RDMA_OFFSET + off);
  62}
  63
  64static inline void rdma_writel(struct bcm_sysport_priv *priv, u32 val, u32 off)
  65{
  66        if (priv->is_lite && off >= RDMA_STATUS)
  67                off += 4;
  68        writel_relaxed(val, priv->base + SYS_PORT_RDMA_OFFSET + off);
  69}
  70
  71static inline u32 tdma_control_bit(struct bcm_sysport_priv *priv, u32 bit)
  72{
  73        if (!priv->is_lite) {
  74                return BIT(bit);
  75        } else {
  76                if (bit >= ACB_ALGO)
  77                        return BIT(bit + 1);
  78                else
  79                        return BIT(bit);
  80        }
  81}
  82
  83/* L2-interrupt masking/unmasking helpers, does automatic saving of the applied
  84 * mask in a software copy to avoid CPU_MASK_STATUS reads in hot-paths.
  85  */
  86#define BCM_SYSPORT_INTR_L2(which)      \
  87static inline void intrl2_##which##_mask_clear(struct bcm_sysport_priv *priv, \
  88                                                u32 mask)               \
  89{                                                                       \
  90        priv->irq##which##_mask &= ~(mask);                             \
  91        intrl2_##which##_writel(priv, mask, INTRL2_CPU_MASK_CLEAR);     \
  92}                                                                       \
  93static inline void intrl2_##which##_mask_set(struct bcm_sysport_priv *priv, \
  94                                                u32 mask)               \
  95{                                                                       \
  96        intrl2_## which##_writel(priv, mask, INTRL2_CPU_MASK_SET);      \
  97        priv->irq##which##_mask |= (mask);                              \
  98}                                                                       \
  99
 100BCM_SYSPORT_INTR_L2(0)
 101BCM_SYSPORT_INTR_L2(1)
 102
 103/* Register accesses to GISB/RBUS registers are expensive (few hundred
 104 * nanoseconds), so keep the check for 64-bits explicit here to save
 105 * one register write per-packet on 32-bits platforms.
 106 */
 107static inline void dma_desc_set_addr(struct bcm_sysport_priv *priv,
 108                                     void __iomem *d,
 109                                     dma_addr_t addr)
 110{
 111#ifdef CONFIG_PHYS_ADDR_T_64BIT
 112        writel_relaxed(upper_32_bits(addr) & DESC_ADDR_HI_MASK,
 113                     d + DESC_ADDR_HI_STATUS_LEN);
 114#endif
 115        writel_relaxed(lower_32_bits(addr), d + DESC_ADDR_LO);
 116}
 117
 118/* Ethtool operations */
 119static void bcm_sysport_set_rx_csum(struct net_device *dev,
 120                                    netdev_features_t wanted)
 121{
 122        struct bcm_sysport_priv *priv = netdev_priv(dev);
 123        u32 reg;
 124
 125        priv->rx_chk_en = !!(wanted & NETIF_F_RXCSUM);
 126        reg = rxchk_readl(priv, RXCHK_CONTROL);
 127        /* Clear L2 header checks, which would prevent BPDUs
 128         * from being received.
 129         */
 130        reg &= ~RXCHK_L2_HDR_DIS;
 131        if (priv->rx_chk_en)
 132                reg |= RXCHK_EN;
 133        else
 134                reg &= ~RXCHK_EN;
 135
 136        /* If UniMAC forwards CRC, we need to skip over it to get
 137         * a valid CHK bit to be set in the per-packet status word
 138         */
 139        if (priv->rx_chk_en && priv->crc_fwd)
 140                reg |= RXCHK_SKIP_FCS;
 141        else
 142                reg &= ~RXCHK_SKIP_FCS;
 143
 144        /* If Broadcom tags are enabled (e.g: using a switch), make
 145         * sure we tell the RXCHK hardware to expect a 4-bytes Broadcom
 146         * tag after the Ethernet MAC Source Address.
 147         */
 148        if (netdev_uses_dsa(dev))
 149                reg |= RXCHK_BRCM_TAG_EN;
 150        else
 151                reg &= ~RXCHK_BRCM_TAG_EN;
 152
 153        rxchk_writel(priv, reg, RXCHK_CONTROL);
 154}
 155
 156static void bcm_sysport_set_tx_csum(struct net_device *dev,
 157                                    netdev_features_t wanted)
 158{
 159        struct bcm_sysport_priv *priv = netdev_priv(dev);
 160        u32 reg;
 161
 162        /* Hardware transmit checksum requires us to enable the Transmit status
 163         * block prepended to the packet contents
 164         */
 165        priv->tsb_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 166                                    NETIF_F_HW_VLAN_CTAG_TX));
 167        reg = tdma_readl(priv, TDMA_CONTROL);
 168        if (priv->tsb_en)
 169                reg |= tdma_control_bit(priv, TSB_EN);
 170        else
 171                reg &= ~tdma_control_bit(priv, TSB_EN);
 172        /* Indicating that software inserts Broadcom tags is needed for the TX
 173         * checksum to be computed correctly when using VLAN HW acceleration,
 174         * else it has no effect, so it can always be turned on.
 175         */
 176        if (netdev_uses_dsa(dev))
 177                reg |= tdma_control_bit(priv, SW_BRCM_TAG);
 178        else
 179                reg &= ~tdma_control_bit(priv, SW_BRCM_TAG);
 180        tdma_writel(priv, reg, TDMA_CONTROL);
 181
 182        /* Default TPID is ETH_P_8021AD, change to ETH_P_8021Q */
 183        if (wanted & NETIF_F_HW_VLAN_CTAG_TX)
 184                tdma_writel(priv, ETH_P_8021Q, TDMA_TPID);
 185}
 186
 187static int bcm_sysport_set_features(struct net_device *dev,
 188                                    netdev_features_t features)
 189{
 190        struct bcm_sysport_priv *priv = netdev_priv(dev);
 191        int ret;
 192
 193        ret = clk_prepare_enable(priv->clk);
 194        if (ret)
 195                return ret;
 196
 197        /* Read CRC forward */
 198        if (!priv->is_lite)
 199                priv->crc_fwd = !!(umac_readl(priv, UMAC_CMD) & CMD_CRC_FWD);
 200        else
 201                priv->crc_fwd = !((gib_readl(priv, GIB_CONTROL) &
 202                                  GIB_FCS_STRIP) >> GIB_FCS_STRIP_SHIFT);
 203
 204        bcm_sysport_set_rx_csum(dev, features);
 205        bcm_sysport_set_tx_csum(dev, features);
 206
 207        clk_disable_unprepare(priv->clk);
 208
 209        return 0;
 210}
 211
 212/* Hardware counters must be kept in sync because the order/offset
 213 * is important here (order in structure declaration = order in hardware)
 214 */
 215static const struct bcm_sysport_stats bcm_sysport_gstrings_stats[] = {
 216        /* general stats */
 217        STAT_NETDEV64(rx_packets),
 218        STAT_NETDEV64(tx_packets),
 219        STAT_NETDEV64(rx_bytes),
 220        STAT_NETDEV64(tx_bytes),
 221        STAT_NETDEV(rx_errors),
 222        STAT_NETDEV(tx_errors),
 223        STAT_NETDEV(rx_dropped),
 224        STAT_NETDEV(tx_dropped),
 225        STAT_NETDEV(multicast),
 226        /* UniMAC RSV counters */
 227        STAT_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
 228        STAT_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
 229        STAT_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
 230        STAT_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
 231        STAT_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
 232        STAT_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
 233        STAT_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
 234        STAT_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
 235        STAT_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
 236        STAT_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
 237        STAT_MIB_RX("rx_pkts", mib.rx.pkt),
 238        STAT_MIB_RX("rx_bytes", mib.rx.bytes),
 239        STAT_MIB_RX("rx_multicast", mib.rx.mca),
 240        STAT_MIB_RX("rx_broadcast", mib.rx.bca),
 241        STAT_MIB_RX("rx_fcs", mib.rx.fcs),
 242        STAT_MIB_RX("rx_control", mib.rx.cf),
 243        STAT_MIB_RX("rx_pause", mib.rx.pf),
 244        STAT_MIB_RX("rx_unknown", mib.rx.uo),
 245        STAT_MIB_RX("rx_align", mib.rx.aln),
 246        STAT_MIB_RX("rx_outrange", mib.rx.flr),
 247        STAT_MIB_RX("rx_code", mib.rx.cde),
 248        STAT_MIB_RX("rx_carrier", mib.rx.fcr),
 249        STAT_MIB_RX("rx_oversize", mib.rx.ovr),
 250        STAT_MIB_RX("rx_jabber", mib.rx.jbr),
 251        STAT_MIB_RX("rx_mtu_err", mib.rx.mtue),
 252        STAT_MIB_RX("rx_good_pkts", mib.rx.pok),
 253        STAT_MIB_RX("rx_unicast", mib.rx.uc),
 254        STAT_MIB_RX("rx_ppp", mib.rx.ppp),
 255        STAT_MIB_RX("rx_crc", mib.rx.rcrc),
 256        /* UniMAC TSV counters */
 257        STAT_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
 258        STAT_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
 259        STAT_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
 260        STAT_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
 261        STAT_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
 262        STAT_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
 263        STAT_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
 264        STAT_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
 265        STAT_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
 266        STAT_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
 267        STAT_MIB_TX("tx_pkts", mib.tx.pkts),
 268        STAT_MIB_TX("tx_multicast", mib.tx.mca),
 269        STAT_MIB_TX("tx_broadcast", mib.tx.bca),
 270        STAT_MIB_TX("tx_pause", mib.tx.pf),
 271        STAT_MIB_TX("tx_control", mib.tx.cf),
 272        STAT_MIB_TX("tx_fcs_err", mib.tx.fcs),
 273        STAT_MIB_TX("tx_oversize", mib.tx.ovr),
 274        STAT_MIB_TX("tx_defer", mib.tx.drf),
 275        STAT_MIB_TX("tx_excess_defer", mib.tx.edf),
 276        STAT_MIB_TX("tx_single_col", mib.tx.scl),
 277        STAT_MIB_TX("tx_multi_col", mib.tx.mcl),
 278        STAT_MIB_TX("tx_late_col", mib.tx.lcl),
 279        STAT_MIB_TX("tx_excess_col", mib.tx.ecl),
 280        STAT_MIB_TX("tx_frags", mib.tx.frg),
 281        STAT_MIB_TX("tx_total_col", mib.tx.ncl),
 282        STAT_MIB_TX("tx_jabber", mib.tx.jbr),
 283        STAT_MIB_TX("tx_bytes", mib.tx.bytes),
 284        STAT_MIB_TX("tx_good_pkts", mib.tx.pok),
 285        STAT_MIB_TX("tx_unicast", mib.tx.uc),
 286        /* UniMAC RUNT counters */
 287        STAT_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
 288        STAT_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
 289        STAT_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
 290        STAT_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
 291        /* RXCHK misc statistics */
 292        STAT_RXCHK("rxchk_bad_csum", mib.rxchk_bad_csum, RXCHK_BAD_CSUM_CNTR),
 293        STAT_RXCHK("rxchk_other_pkt_disc", mib.rxchk_other_pkt_disc,
 294                   RXCHK_OTHER_DISC_CNTR),
 295        /* RBUF misc statistics */
 296        STAT_RBUF("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt, RBUF_OVFL_DISC_CNTR),
 297        STAT_RBUF("rbuf_err_cnt", mib.rbuf_err_cnt, RBUF_ERR_PKT_CNTR),
 298        STAT_MIB_SOFT("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
 299        STAT_MIB_SOFT("rx_dma_failed", mib.rx_dma_failed),
 300        STAT_MIB_SOFT("tx_dma_failed", mib.tx_dma_failed),
 301        STAT_MIB_SOFT("tx_realloc_tsb", mib.tx_realloc_tsb),
 302        STAT_MIB_SOFT("tx_realloc_tsb_failed", mib.tx_realloc_tsb_failed),
 303        /* Per TX-queue statistics are dynamically appended */
 304};
 305
 306#define BCM_SYSPORT_STATS_LEN   ARRAY_SIZE(bcm_sysport_gstrings_stats)
 307
 308static void bcm_sysport_get_drvinfo(struct net_device *dev,
 309                                    struct ethtool_drvinfo *info)
 310{
 311        strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 312        strlcpy(info->bus_info, "platform", sizeof(info->bus_info));
 313}
 314
 315static u32 bcm_sysport_get_msglvl(struct net_device *dev)
 316{
 317        struct bcm_sysport_priv *priv = netdev_priv(dev);
 318
 319        return priv->msg_enable;
 320}
 321
 322static void bcm_sysport_set_msglvl(struct net_device *dev, u32 enable)
 323{
 324        struct bcm_sysport_priv *priv = netdev_priv(dev);
 325
 326        priv->msg_enable = enable;
 327}
 328
 329static inline bool bcm_sysport_lite_stat_valid(enum bcm_sysport_stat_type type)
 330{
 331        switch (type) {
 332        case BCM_SYSPORT_STAT_NETDEV:
 333        case BCM_SYSPORT_STAT_NETDEV64:
 334        case BCM_SYSPORT_STAT_RXCHK:
 335        case BCM_SYSPORT_STAT_RBUF:
 336        case BCM_SYSPORT_STAT_SOFT:
 337                return true;
 338        default:
 339                return false;
 340        }
 341}
 342
 343static int bcm_sysport_get_sset_count(struct net_device *dev, int string_set)
 344{
 345        struct bcm_sysport_priv *priv = netdev_priv(dev);
 346        const struct bcm_sysport_stats *s;
 347        unsigned int i, j;
 348
 349        switch (string_set) {
 350        case ETH_SS_STATS:
 351                for (i = 0, j = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
 352                        s = &bcm_sysport_gstrings_stats[i];
 353                        if (priv->is_lite &&
 354                            !bcm_sysport_lite_stat_valid(s->type))
 355                                continue;
 356                        j++;
 357                }
 358                /* Include per-queue statistics */
 359                return j + dev->num_tx_queues * NUM_SYSPORT_TXQ_STAT;
 360        default:
 361                return -EOPNOTSUPP;
 362        }
 363}
 364
 365static void bcm_sysport_get_strings(struct net_device *dev,
 366                                    u32 stringset, u8 *data)
 367{
 368        struct bcm_sysport_priv *priv = netdev_priv(dev);
 369        const struct bcm_sysport_stats *s;
 370        char buf[128];
 371        int i, j;
 372
 373        switch (stringset) {
 374        case ETH_SS_STATS:
 375                for (i = 0, j = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
 376                        s = &bcm_sysport_gstrings_stats[i];
 377                        if (priv->is_lite &&
 378                            !bcm_sysport_lite_stat_valid(s->type))
 379                                continue;
 380
 381                        memcpy(data + j * ETH_GSTRING_LEN, s->stat_string,
 382                               ETH_GSTRING_LEN);
 383                        j++;
 384                }
 385
 386                for (i = 0; i < dev->num_tx_queues; i++) {
 387                        snprintf(buf, sizeof(buf), "txq%d_packets", i);
 388                        memcpy(data + j * ETH_GSTRING_LEN, buf,
 389                               ETH_GSTRING_LEN);
 390                        j++;
 391
 392                        snprintf(buf, sizeof(buf), "txq%d_bytes", i);
 393                        memcpy(data + j * ETH_GSTRING_LEN, buf,
 394                               ETH_GSTRING_LEN);
 395                        j++;
 396                }
 397                break;
 398        default:
 399                break;
 400        }
 401}
 402
 403static void bcm_sysport_update_mib_counters(struct bcm_sysport_priv *priv)
 404{
 405        int i, j = 0;
 406
 407        for (i = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
 408                const struct bcm_sysport_stats *s;
 409                u8 offset = 0;
 410                u32 val = 0;
 411                char *p;
 412
 413                s = &bcm_sysport_gstrings_stats[i];
 414                switch (s->type) {
 415                case BCM_SYSPORT_STAT_NETDEV:
 416                case BCM_SYSPORT_STAT_NETDEV64:
 417                case BCM_SYSPORT_STAT_SOFT:
 418                        continue;
 419                case BCM_SYSPORT_STAT_MIB_RX:
 420                case BCM_SYSPORT_STAT_MIB_TX:
 421                case BCM_SYSPORT_STAT_RUNT:
 422                        if (priv->is_lite)
 423                                continue;
 424
 425                        if (s->type != BCM_SYSPORT_STAT_MIB_RX)
 426                                offset = UMAC_MIB_STAT_OFFSET;
 427                        val = umac_readl(priv, UMAC_MIB_START + j + offset);
 428                        break;
 429                case BCM_SYSPORT_STAT_RXCHK:
 430                        val = rxchk_readl(priv, s->reg_offset);
 431                        if (val == ~0)
 432                                rxchk_writel(priv, 0, s->reg_offset);
 433                        break;
 434                case BCM_SYSPORT_STAT_RBUF:
 435                        val = rbuf_readl(priv, s->reg_offset);
 436                        if (val == ~0)
 437                                rbuf_writel(priv, 0, s->reg_offset);
 438                        break;
 439                }
 440
 441                j += s->stat_sizeof;
 442                p = (char *)priv + s->stat_offset;
 443                *(u32 *)p = val;
 444        }
 445
 446        netif_dbg(priv, hw, priv->netdev, "updated MIB counters\n");
 447}
 448
 449static void bcm_sysport_update_tx_stats(struct bcm_sysport_priv *priv,
 450                                        u64 *tx_bytes, u64 *tx_packets)
 451{
 452        struct bcm_sysport_tx_ring *ring;
 453        u64 bytes = 0, packets = 0;
 454        unsigned int start;
 455        unsigned int q;
 456
 457        for (q = 0; q < priv->netdev->num_tx_queues; q++) {
 458                ring = &priv->tx_rings[q];
 459                do {
 460                        start = u64_stats_fetch_begin_irq(&priv->syncp);
 461                        bytes = ring->bytes;
 462                        packets = ring->packets;
 463                } while (u64_stats_fetch_retry_irq(&priv->syncp, start));
 464
 465                *tx_bytes += bytes;
 466                *tx_packets += packets;
 467        }
 468}
 469
 470static void bcm_sysport_get_stats(struct net_device *dev,
 471                                  struct ethtool_stats *stats, u64 *data)
 472{
 473        struct bcm_sysport_priv *priv = netdev_priv(dev);
 474        struct bcm_sysport_stats64 *stats64 = &priv->stats64;
 475        struct u64_stats_sync *syncp = &priv->syncp;
 476        struct bcm_sysport_tx_ring *ring;
 477        u64 tx_bytes = 0, tx_packets = 0;
 478        unsigned int start;
 479        int i, j;
 480
 481        if (netif_running(dev)) {
 482                bcm_sysport_update_mib_counters(priv);
 483                bcm_sysport_update_tx_stats(priv, &tx_bytes, &tx_packets);
 484                stats64->tx_bytes = tx_bytes;
 485                stats64->tx_packets = tx_packets;
 486        }
 487
 488        for (i =  0, j = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
 489                const struct bcm_sysport_stats *s;
 490                char *p;
 491
 492                s = &bcm_sysport_gstrings_stats[i];
 493                if (s->type == BCM_SYSPORT_STAT_NETDEV)
 494                        p = (char *)&dev->stats;
 495                else if (s->type == BCM_SYSPORT_STAT_NETDEV64)
 496                        p = (char *)stats64;
 497                else
 498                        p = (char *)priv;
 499
 500                if (priv->is_lite && !bcm_sysport_lite_stat_valid(s->type))
 501                        continue;
 502                p += s->stat_offset;
 503
 504                if (s->stat_sizeof == sizeof(u64) &&
 505                    s->type == BCM_SYSPORT_STAT_NETDEV64) {
 506                        do {
 507                                start = u64_stats_fetch_begin_irq(syncp);
 508                                data[i] = *(u64 *)p;
 509                        } while (u64_stats_fetch_retry_irq(syncp, start));
 510                } else
 511                        data[i] = *(u32 *)p;
 512                j++;
 513        }
 514
 515        /* For SYSTEMPORT Lite since we have holes in our statistics, j would
 516         * be equal to BCM_SYSPORT_STATS_LEN at the end of the loop, but it
 517         * needs to point to how many total statistics we have minus the
 518         * number of per TX queue statistics
 519         */
 520        j = bcm_sysport_get_sset_count(dev, ETH_SS_STATS) -
 521            dev->num_tx_queues * NUM_SYSPORT_TXQ_STAT;
 522
 523        for (i = 0; i < dev->num_tx_queues; i++) {
 524                ring = &priv->tx_rings[i];
 525                data[j] = ring->packets;
 526                j++;
 527                data[j] = ring->bytes;
 528                j++;
 529        }
 530}
 531
 532static void bcm_sysport_get_wol(struct net_device *dev,
 533                                struct ethtool_wolinfo *wol)
 534{
 535        struct bcm_sysport_priv *priv = netdev_priv(dev);
 536
 537        wol->supported = WAKE_MAGIC | WAKE_MAGICSECURE | WAKE_FILTER;
 538        wol->wolopts = priv->wolopts;
 539
 540        if (!(priv->wolopts & WAKE_MAGICSECURE))
 541                return;
 542
 543        memcpy(wol->sopass, priv->sopass, sizeof(priv->sopass));
 544}
 545
 546static int bcm_sysport_set_wol(struct net_device *dev,
 547                               struct ethtool_wolinfo *wol)
 548{
 549        struct bcm_sysport_priv *priv = netdev_priv(dev);
 550        struct device *kdev = &priv->pdev->dev;
 551        u32 supported = WAKE_MAGIC | WAKE_MAGICSECURE | WAKE_FILTER;
 552
 553        if (!device_can_wakeup(kdev))
 554                return -ENOTSUPP;
 555
 556        if (wol->wolopts & ~supported)
 557                return -EINVAL;
 558
 559        if (wol->wolopts & WAKE_MAGICSECURE)
 560                memcpy(priv->sopass, wol->sopass, sizeof(priv->sopass));
 561
 562        /* Flag the device and relevant IRQ as wakeup capable */
 563        if (wol->wolopts) {
 564                device_set_wakeup_enable(kdev, 1);
 565                if (priv->wol_irq_disabled)
 566                        enable_irq_wake(priv->wol_irq);
 567                priv->wol_irq_disabled = 0;
 568        } else {
 569                device_set_wakeup_enable(kdev, 0);
 570                /* Avoid unbalanced disable_irq_wake calls */
 571                if (!priv->wol_irq_disabled)
 572                        disable_irq_wake(priv->wol_irq);
 573                priv->wol_irq_disabled = 1;
 574        }
 575
 576        priv->wolopts = wol->wolopts;
 577
 578        return 0;
 579}
 580
 581static void bcm_sysport_set_rx_coalesce(struct bcm_sysport_priv *priv,
 582                                        u32 usecs, u32 pkts)
 583{
 584        u32 reg;
 585
 586        reg = rdma_readl(priv, RDMA_MBDONE_INTR);
 587        reg &= ~(RDMA_INTR_THRESH_MASK |
 588                 RDMA_TIMEOUT_MASK << RDMA_TIMEOUT_SHIFT);
 589        reg |= pkts;
 590        reg |= DIV_ROUND_UP(usecs * 1000, 8192) << RDMA_TIMEOUT_SHIFT;
 591        rdma_writel(priv, reg, RDMA_MBDONE_INTR);
 592}
 593
 594static void bcm_sysport_set_tx_coalesce(struct bcm_sysport_tx_ring *ring,
 595                                        struct ethtool_coalesce *ec)
 596{
 597        struct bcm_sysport_priv *priv = ring->priv;
 598        u32 reg;
 599
 600        reg = tdma_readl(priv, TDMA_DESC_RING_INTR_CONTROL(ring->index));
 601        reg &= ~(RING_INTR_THRESH_MASK |
 602                 RING_TIMEOUT_MASK << RING_TIMEOUT_SHIFT);
 603        reg |= ec->tx_max_coalesced_frames;
 604        reg |= DIV_ROUND_UP(ec->tx_coalesce_usecs * 1000, 8192) <<
 605                            RING_TIMEOUT_SHIFT;
 606        tdma_writel(priv, reg, TDMA_DESC_RING_INTR_CONTROL(ring->index));
 607}
 608
 609static int bcm_sysport_get_coalesce(struct net_device *dev,
 610                                    struct ethtool_coalesce *ec)
 611{
 612        struct bcm_sysport_priv *priv = netdev_priv(dev);
 613        u32 reg;
 614
 615        reg = tdma_readl(priv, TDMA_DESC_RING_INTR_CONTROL(0));
 616
 617        ec->tx_coalesce_usecs = (reg >> RING_TIMEOUT_SHIFT) * 8192 / 1000;
 618        ec->tx_max_coalesced_frames = reg & RING_INTR_THRESH_MASK;
 619
 620        reg = rdma_readl(priv, RDMA_MBDONE_INTR);
 621
 622        ec->rx_coalesce_usecs = (reg >> RDMA_TIMEOUT_SHIFT) * 8192 / 1000;
 623        ec->rx_max_coalesced_frames = reg & RDMA_INTR_THRESH_MASK;
 624        ec->use_adaptive_rx_coalesce = priv->dim.use_dim;
 625
 626        return 0;
 627}
 628
 629static int bcm_sysport_set_coalesce(struct net_device *dev,
 630                                    struct ethtool_coalesce *ec)
 631{
 632        struct bcm_sysport_priv *priv = netdev_priv(dev);
 633        struct dim_cq_moder moder;
 634        u32 usecs, pkts;
 635        unsigned int i;
 636
 637        /* Base system clock is 125Mhz, DMA timeout is this reference clock
 638         * divided by 1024, which yield roughly 8.192 us, our maximum value has
 639         * to fit in the RING_TIMEOUT_MASK (16 bits).
 640         */
 641        if (ec->tx_max_coalesced_frames > RING_INTR_THRESH_MASK ||
 642            ec->tx_coalesce_usecs > (RING_TIMEOUT_MASK * 8) + 1 ||
 643            ec->rx_max_coalesced_frames > RDMA_INTR_THRESH_MASK ||
 644            ec->rx_coalesce_usecs > (RDMA_TIMEOUT_MASK * 8) + 1)
 645                return -EINVAL;
 646
 647        if ((ec->tx_coalesce_usecs == 0 && ec->tx_max_coalesced_frames == 0) ||
 648            (ec->rx_coalesce_usecs == 0 && ec->rx_max_coalesced_frames == 0))
 649                return -EINVAL;
 650
 651        for (i = 0; i < dev->num_tx_queues; i++)
 652                bcm_sysport_set_tx_coalesce(&priv->tx_rings[i], ec);
 653
 654        priv->rx_coalesce_usecs = ec->rx_coalesce_usecs;
 655        priv->rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
 656        usecs = priv->rx_coalesce_usecs;
 657        pkts = priv->rx_max_coalesced_frames;
 658
 659        if (ec->use_adaptive_rx_coalesce && !priv->dim.use_dim) {
 660                moder = net_dim_get_def_rx_moderation(priv->dim.dim.mode);
 661                usecs = moder.usec;
 662                pkts = moder.pkts;
 663        }
 664
 665        priv->dim.use_dim = ec->use_adaptive_rx_coalesce;
 666
 667        /* Apply desired coalescing parameters */
 668        bcm_sysport_set_rx_coalesce(priv, usecs, pkts);
 669
 670        return 0;
 671}
 672
 673static void bcm_sysport_free_cb(struct bcm_sysport_cb *cb)
 674{
 675        dev_consume_skb_any(cb->skb);
 676        cb->skb = NULL;
 677        dma_unmap_addr_set(cb, dma_addr, 0);
 678}
 679
 680static struct sk_buff *bcm_sysport_rx_refill(struct bcm_sysport_priv *priv,
 681                                             struct bcm_sysport_cb *cb)
 682{
 683        struct device *kdev = &priv->pdev->dev;
 684        struct net_device *ndev = priv->netdev;
 685        struct sk_buff *skb, *rx_skb;
 686        dma_addr_t mapping;
 687
 688        /* Allocate a new SKB for a new packet */
 689        skb = __netdev_alloc_skb(priv->netdev, RX_BUF_LENGTH,
 690                                 GFP_ATOMIC | __GFP_NOWARN);
 691        if (!skb) {
 692                priv->mib.alloc_rx_buff_failed++;
 693                netif_err(priv, rx_err, ndev, "SKB alloc failed\n");
 694                return NULL;
 695        }
 696
 697        mapping = dma_map_single(kdev, skb->data,
 698                                 RX_BUF_LENGTH, DMA_FROM_DEVICE);
 699        if (dma_mapping_error(kdev, mapping)) {
 700                priv->mib.rx_dma_failed++;
 701                dev_kfree_skb_any(skb);
 702                netif_err(priv, rx_err, ndev, "DMA mapping failure\n");
 703                return NULL;
 704        }
 705
 706        /* Grab the current SKB on the ring */
 707        rx_skb = cb->skb;
 708        if (likely(rx_skb))
 709                dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
 710                                 RX_BUF_LENGTH, DMA_FROM_DEVICE);
 711
 712        /* Put the new SKB on the ring */
 713        cb->skb = skb;
 714        dma_unmap_addr_set(cb, dma_addr, mapping);
 715        dma_desc_set_addr(priv, cb->bd_addr, mapping);
 716
 717        netif_dbg(priv, rx_status, ndev, "RX refill\n");
 718
 719        /* Return the current SKB to the caller */
 720        return rx_skb;
 721}
 722
 723static int bcm_sysport_alloc_rx_bufs(struct bcm_sysport_priv *priv)
 724{
 725        struct bcm_sysport_cb *cb;
 726        struct sk_buff *skb;
 727        unsigned int i;
 728
 729        for (i = 0; i < priv->num_rx_bds; i++) {
 730                cb = &priv->rx_cbs[i];
 731                skb = bcm_sysport_rx_refill(priv, cb);
 732                dev_kfree_skb(skb);
 733                if (!cb->skb)
 734                        return -ENOMEM;
 735        }
 736
 737        return 0;
 738}
 739
 740/* Poll the hardware for up to budget packets to process */
 741static unsigned int bcm_sysport_desc_rx(struct bcm_sysport_priv *priv,
 742                                        unsigned int budget)
 743{
 744        struct bcm_sysport_stats64 *stats64 = &priv->stats64;
 745        struct net_device *ndev = priv->netdev;
 746        unsigned int processed = 0, to_process;
 747        unsigned int processed_bytes = 0;
 748        struct bcm_sysport_cb *cb;
 749        struct sk_buff *skb;
 750        unsigned int p_index;
 751        u16 len, status;
 752        struct bcm_rsb *rsb;
 753
 754        /* Clear status before servicing to reduce spurious interrupts */
 755        intrl2_0_writel(priv, INTRL2_0_RDMA_MBDONE, INTRL2_CPU_CLEAR);
 756
 757        /* Determine how much we should process since last call, SYSTEMPORT Lite
 758         * groups the producer and consumer indexes into the same 32-bit
 759         * which we access using RDMA_CONS_INDEX
 760         */
 761        if (!priv->is_lite)
 762                p_index = rdma_readl(priv, RDMA_PROD_INDEX);
 763        else
 764                p_index = rdma_readl(priv, RDMA_CONS_INDEX);
 765        p_index &= RDMA_PROD_INDEX_MASK;
 766
 767        to_process = (p_index - priv->rx_c_index) & RDMA_CONS_INDEX_MASK;
 768
 769        netif_dbg(priv, rx_status, ndev,
 770                  "p_index=%d rx_c_index=%d to_process=%d\n",
 771                  p_index, priv->rx_c_index, to_process);
 772
 773        while ((processed < to_process) && (processed < budget)) {
 774                cb = &priv->rx_cbs[priv->rx_read_ptr];
 775                skb = bcm_sysport_rx_refill(priv, cb);
 776
 777
 778                /* We do not have a backing SKB, so we do not a corresponding
 779                 * DMA mapping for this incoming packet since
 780                 * bcm_sysport_rx_refill always either has both skb and mapping
 781                 * or none.
 782                 */
 783                if (unlikely(!skb)) {
 784                        netif_err(priv, rx_err, ndev, "out of memory!\n");
 785                        ndev->stats.rx_dropped++;
 786                        ndev->stats.rx_errors++;
 787                        goto next;
 788                }
 789
 790                /* Extract the Receive Status Block prepended */
 791                rsb = (struct bcm_rsb *)skb->data;
 792                len = (rsb->rx_status_len >> DESC_LEN_SHIFT) & DESC_LEN_MASK;
 793                status = (rsb->rx_status_len >> DESC_STATUS_SHIFT) &
 794                          DESC_STATUS_MASK;
 795
 796                netif_dbg(priv, rx_status, ndev,
 797                          "p=%d, c=%d, rd_ptr=%d, len=%d, flag=0x%04x\n",
 798                          p_index, priv->rx_c_index, priv->rx_read_ptr,
 799                          len, status);
 800
 801                if (unlikely(len > RX_BUF_LENGTH)) {
 802                        netif_err(priv, rx_status, ndev, "oversized packet\n");
 803                        ndev->stats.rx_length_errors++;
 804                        ndev->stats.rx_errors++;
 805                        dev_kfree_skb_any(skb);
 806                        goto next;
 807                }
 808
 809                if (unlikely(!(status & DESC_EOP) || !(status & DESC_SOP))) {
 810                        netif_err(priv, rx_status, ndev, "fragmented packet!\n");
 811                        ndev->stats.rx_dropped++;
 812                        ndev->stats.rx_errors++;
 813                        dev_kfree_skb_any(skb);
 814                        goto next;
 815                }
 816
 817                if (unlikely(status & (RX_STATUS_ERR | RX_STATUS_OVFLOW))) {
 818                        netif_err(priv, rx_err, ndev, "error packet\n");
 819                        if (status & RX_STATUS_OVFLOW)
 820                                ndev->stats.rx_over_errors++;
 821                        ndev->stats.rx_dropped++;
 822                        ndev->stats.rx_errors++;
 823                        dev_kfree_skb_any(skb);
 824                        goto next;
 825                }
 826
 827                skb_put(skb, len);
 828
 829                /* Hardware validated our checksum */
 830                if (likely(status & DESC_L4_CSUM))
 831                        skb->ip_summed = CHECKSUM_UNNECESSARY;
 832
 833                /* Hardware pre-pends packets with 2bytes before Ethernet
 834                 * header plus we have the Receive Status Block, strip off all
 835                 * of this from the SKB.
 836                 */
 837                skb_pull(skb, sizeof(*rsb) + 2);
 838                len -= (sizeof(*rsb) + 2);
 839                processed_bytes += len;
 840
 841                /* UniMAC may forward CRC */
 842                if (priv->crc_fwd) {
 843                        skb_trim(skb, len - ETH_FCS_LEN);
 844                        len -= ETH_FCS_LEN;
 845                }
 846
 847                skb->protocol = eth_type_trans(skb, ndev);
 848                ndev->stats.rx_packets++;
 849                ndev->stats.rx_bytes += len;
 850                u64_stats_update_begin(&priv->syncp);
 851                stats64->rx_packets++;
 852                stats64->rx_bytes += len;
 853                u64_stats_update_end(&priv->syncp);
 854
 855                napi_gro_receive(&priv->napi, skb);
 856next:
 857                processed++;
 858                priv->rx_read_ptr++;
 859
 860                if (priv->rx_read_ptr == priv->num_rx_bds)
 861                        priv->rx_read_ptr = 0;
 862        }
 863
 864        priv->dim.packets = processed;
 865        priv->dim.bytes = processed_bytes;
 866
 867        return processed;
 868}
 869
 870static void bcm_sysport_tx_reclaim_one(struct bcm_sysport_tx_ring *ring,
 871                                       struct bcm_sysport_cb *cb,
 872                                       unsigned int *bytes_compl,
 873                                       unsigned int *pkts_compl)
 874{
 875        struct bcm_sysport_priv *priv = ring->priv;
 876        struct device *kdev = &priv->pdev->dev;
 877
 878        if (cb->skb) {
 879                *bytes_compl += cb->skb->len;
 880                dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
 881                                 dma_unmap_len(cb, dma_len),
 882                                 DMA_TO_DEVICE);
 883                (*pkts_compl)++;
 884                bcm_sysport_free_cb(cb);
 885        /* SKB fragment */
 886        } else if (dma_unmap_addr(cb, dma_addr)) {
 887                *bytes_compl += dma_unmap_len(cb, dma_len);
 888                dma_unmap_page(kdev, dma_unmap_addr(cb, dma_addr),
 889                               dma_unmap_len(cb, dma_len), DMA_TO_DEVICE);
 890                dma_unmap_addr_set(cb, dma_addr, 0);
 891        }
 892}
 893
 894/* Reclaim queued SKBs for transmission completion, lockless version */
 895static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
 896                                             struct bcm_sysport_tx_ring *ring)
 897{
 898        unsigned int pkts_compl = 0, bytes_compl = 0;
 899        struct net_device *ndev = priv->netdev;
 900        unsigned int txbds_processed = 0;
 901        struct bcm_sysport_cb *cb;
 902        unsigned int txbds_ready;
 903        unsigned int c_index;
 904        u32 hw_ind;
 905
 906        /* Clear status before servicing to reduce spurious interrupts */
 907        if (!ring->priv->is_lite)
 908                intrl2_1_writel(ring->priv, BIT(ring->index), INTRL2_CPU_CLEAR);
 909        else
 910                intrl2_0_writel(ring->priv, BIT(ring->index +
 911                                INTRL2_0_TDMA_MBDONE_SHIFT), INTRL2_CPU_CLEAR);
 912
 913        /* Compute how many descriptors have been processed since last call */
 914        hw_ind = tdma_readl(priv, TDMA_DESC_RING_PROD_CONS_INDEX(ring->index));
 915        c_index = (hw_ind >> RING_CONS_INDEX_SHIFT) & RING_CONS_INDEX_MASK;
 916        txbds_ready = (c_index - ring->c_index) & RING_CONS_INDEX_MASK;
 917
 918        netif_dbg(priv, tx_done, ndev,
 919                  "ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
 920                  ring->index, ring->c_index, c_index, txbds_ready);
 921
 922        while (txbds_processed < txbds_ready) {
 923                cb = &ring->cbs[ring->clean_index];
 924                bcm_sysport_tx_reclaim_one(ring, cb, &bytes_compl, &pkts_compl);
 925
 926                ring->desc_count++;
 927                txbds_processed++;
 928
 929                if (likely(ring->clean_index < ring->size - 1))
 930                        ring->clean_index++;
 931                else
 932                        ring->clean_index = 0;
 933        }
 934
 935        u64_stats_update_begin(&priv->syncp);
 936        ring->packets += pkts_compl;
 937        ring->bytes += bytes_compl;
 938        u64_stats_update_end(&priv->syncp);
 939
 940        ring->c_index = c_index;
 941
 942        netif_dbg(priv, tx_done, ndev,
 943                  "ring=%d c_index=%d pkts_compl=%d, bytes_compl=%d\n",
 944                  ring->index, ring->c_index, pkts_compl, bytes_compl);
 945
 946        return pkts_compl;
 947}
 948
 949/* Locked version of the per-ring TX reclaim routine */
 950static unsigned int bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
 951                                           struct bcm_sysport_tx_ring *ring)
 952{
 953        struct netdev_queue *txq;
 954        unsigned int released;
 955        unsigned long flags;
 956
 957        txq = netdev_get_tx_queue(priv->netdev, ring->index);
 958
 959        spin_lock_irqsave(&ring->lock, flags);
 960        released = __bcm_sysport_tx_reclaim(priv, ring);
 961        if (released)
 962                netif_tx_wake_queue(txq);
 963
 964        spin_unlock_irqrestore(&ring->lock, flags);
 965
 966        return released;
 967}
 968
 969/* Locked version of the per-ring TX reclaim, but does not wake the queue */
 970static void bcm_sysport_tx_clean(struct bcm_sysport_priv *priv,
 971                                 struct bcm_sysport_tx_ring *ring)
 972{
 973        unsigned long flags;
 974
 975        spin_lock_irqsave(&ring->lock, flags);
 976        __bcm_sysport_tx_reclaim(priv, ring);
 977        spin_unlock_irqrestore(&ring->lock, flags);
 978}
 979
 980static int bcm_sysport_tx_poll(struct napi_struct *napi, int budget)
 981{
 982        struct bcm_sysport_tx_ring *ring =
 983                container_of(napi, struct bcm_sysport_tx_ring, napi);
 984        unsigned int work_done = 0;
 985
 986        work_done = bcm_sysport_tx_reclaim(ring->priv, ring);
 987
 988        if (work_done == 0) {
 989                napi_complete(napi);
 990                /* re-enable TX interrupt */
 991                if (!ring->priv->is_lite)
 992                        intrl2_1_mask_clear(ring->priv, BIT(ring->index));
 993                else
 994                        intrl2_0_mask_clear(ring->priv, BIT(ring->index +
 995                                            INTRL2_0_TDMA_MBDONE_SHIFT));
 996
 997                return 0;
 998        }
 999
1000        return budget;
1001}
1002
1003static void bcm_sysport_tx_reclaim_all(struct bcm_sysport_priv *priv)
1004{
1005        unsigned int q;
1006
1007        for (q = 0; q < priv->netdev->num_tx_queues; q++)
1008                bcm_sysport_tx_reclaim(priv, &priv->tx_rings[q]);
1009}
1010
1011static int bcm_sysport_poll(struct napi_struct *napi, int budget)
1012{
1013        struct bcm_sysport_priv *priv =
1014                container_of(napi, struct bcm_sysport_priv, napi);
1015        struct dim_sample dim_sample = {};
1016        unsigned int work_done = 0;
1017
1018        work_done = bcm_sysport_desc_rx(priv, budget);
1019
1020        priv->rx_c_index += work_done;
1021        priv->rx_c_index &= RDMA_CONS_INDEX_MASK;
1022
1023        /* SYSTEMPORT Lite groups the producer/consumer index, producer is
1024         * maintained by HW, but writes to it will be ignore while RDMA
1025         * is active
1026         */
1027        if (!priv->is_lite)
1028                rdma_writel(priv, priv->rx_c_index, RDMA_CONS_INDEX);
1029        else
1030                rdma_writel(priv, priv->rx_c_index << 16, RDMA_CONS_INDEX);
1031
1032        if (work_done < budget) {
1033                napi_complete_done(napi, work_done);
1034                /* re-enable RX interrupts */
1035                intrl2_0_mask_clear(priv, INTRL2_0_RDMA_MBDONE);
1036        }
1037
1038        if (priv->dim.use_dim) {
1039                dim_update_sample(priv->dim.event_ctr, priv->dim.packets,
1040                                  priv->dim.bytes, &dim_sample);
1041                net_dim(&priv->dim.dim, dim_sample);
1042        }
1043
1044        return work_done;
1045}
1046
1047static void mpd_enable_set(struct bcm_sysport_priv *priv, bool enable)
1048{
1049        u32 reg, bit;
1050
1051        reg = umac_readl(priv, UMAC_MPD_CTRL);
1052        if (enable)
1053                reg |= MPD_EN;
1054        else
1055                reg &= ~MPD_EN;
1056        umac_writel(priv, reg, UMAC_MPD_CTRL);
1057
1058        if (priv->is_lite)
1059                bit = RBUF_ACPI_EN_LITE;
1060        else
1061                bit = RBUF_ACPI_EN;
1062
1063        reg = rbuf_readl(priv, RBUF_CONTROL);
1064        if (enable)
1065                reg |= bit;
1066        else
1067                reg &= ~bit;
1068        rbuf_writel(priv, reg, RBUF_CONTROL);
1069}
1070
1071static void bcm_sysport_resume_from_wol(struct bcm_sysport_priv *priv)
1072{
1073        unsigned int index;
1074        u32 reg;
1075
1076        /* Disable RXCHK, active filters and Broadcom tag matching */
1077        reg = rxchk_readl(priv, RXCHK_CONTROL);
1078        reg &= ~(RXCHK_BRCM_TAG_MATCH_MASK <<
1079                 RXCHK_BRCM_TAG_MATCH_SHIFT | RXCHK_EN | RXCHK_BRCM_TAG_EN);
1080        rxchk_writel(priv, reg, RXCHK_CONTROL);
1081
1082        /* Make sure we restore correct CID index in case HW lost
1083         * its context during deep idle state
1084         */
1085        for_each_set_bit(index, priv->filters, RXCHK_BRCM_TAG_MAX) {
1086                rxchk_writel(priv, priv->filters_loc[index] <<
1087                             RXCHK_BRCM_TAG_CID_SHIFT, RXCHK_BRCM_TAG(index));
1088                rxchk_writel(priv, 0xff00ffff, RXCHK_BRCM_TAG_MASK(index));
1089        }
1090
1091        /* Clear the MagicPacket detection logic */
1092        mpd_enable_set(priv, false);
1093
1094        reg = intrl2_0_readl(priv, INTRL2_CPU_STATUS);
1095        if (reg & INTRL2_0_MPD)
1096                netdev_info(priv->netdev, "Wake-on-LAN (MPD) interrupt!\n");
1097
1098        if (reg & INTRL2_0_BRCM_MATCH_TAG) {
1099                reg = rxchk_readl(priv, RXCHK_BRCM_TAG_MATCH_STATUS) &
1100                                  RXCHK_BRCM_TAG_MATCH_MASK;
1101                netdev_info(priv->netdev,
1102                            "Wake-on-LAN (filters 0x%02x) interrupt!\n", reg);
1103        }
1104
1105        netif_dbg(priv, wol, priv->netdev, "resumed from WOL\n");
1106}
1107
1108static void bcm_sysport_dim_work(struct work_struct *work)
1109{
1110        struct dim *dim = container_of(work, struct dim, work);
1111        struct bcm_sysport_net_dim *ndim =
1112                        container_of(dim, struct bcm_sysport_net_dim, dim);
1113        struct bcm_sysport_priv *priv =
1114                        container_of(ndim, struct bcm_sysport_priv, dim);
1115        struct dim_cq_moder cur_profile = net_dim_get_rx_moderation(dim->mode,
1116                                                                    dim->profile_ix);
1117
1118        bcm_sysport_set_rx_coalesce(priv, cur_profile.usec, cur_profile.pkts);
1119        dim->state = DIM_START_MEASURE;
1120}
1121
1122/* RX and misc interrupt routine */
1123static irqreturn_t bcm_sysport_rx_isr(int irq, void *dev_id)
1124{
1125        struct net_device *dev = dev_id;
1126        struct bcm_sysport_priv *priv = netdev_priv(dev);
1127        struct bcm_sysport_tx_ring *txr;
1128        unsigned int ring, ring_bit;
1129
1130        priv->irq0_stat = intrl2_0_readl(priv, INTRL2_CPU_STATUS) &
1131                          ~intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
1132        intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR);
1133
1134        if (unlikely(priv->irq0_stat == 0)) {
1135                netdev_warn(priv->netdev, "spurious RX interrupt\n");
1136                return IRQ_NONE;
1137        }
1138
1139        if (priv->irq0_stat & INTRL2_0_RDMA_MBDONE) {
1140                priv->dim.event_ctr++;
1141                if (likely(napi_schedule_prep(&priv->napi))) {
1142                        /* disable RX interrupts */
1143                        intrl2_0_mask_set(priv, INTRL2_0_RDMA_MBDONE);
1144                        __napi_schedule_irqoff(&priv->napi);
1145                }
1146        }
1147
1148        /* TX ring is full, perform a full reclaim since we do not know
1149         * which one would trigger this interrupt
1150         */
1151        if (priv->irq0_stat & INTRL2_0_TX_RING_FULL)
1152                bcm_sysport_tx_reclaim_all(priv);
1153
1154        if (!priv->is_lite)
1155                goto out;
1156
1157        for (ring = 0; ring < dev->num_tx_queues; ring++) {
1158                ring_bit = BIT(ring + INTRL2_0_TDMA_MBDONE_SHIFT);
1159                if (!(priv->irq0_stat & ring_bit))
1160                        continue;
1161
1162                txr = &priv->tx_rings[ring];
1163
1164                if (likely(napi_schedule_prep(&txr->napi))) {
1165                        intrl2_0_mask_set(priv, ring_bit);
1166                        __napi_schedule(&txr->napi);
1167                }
1168        }
1169out:
1170        return IRQ_HANDLED;
1171}
1172
1173/* TX interrupt service routine */
1174static irqreturn_t bcm_sysport_tx_isr(int irq, void *dev_id)
1175{
1176        struct net_device *dev = dev_id;
1177        struct bcm_sysport_priv *priv = netdev_priv(dev);
1178        struct bcm_sysport_tx_ring *txr;
1179        unsigned int ring;
1180
1181        priv->irq1_stat = intrl2_1_readl(priv, INTRL2_CPU_STATUS) &
1182                                ~intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
1183        intrl2_1_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
1184
1185        if (unlikely(priv->irq1_stat == 0)) {
1186                netdev_warn(priv->netdev, "spurious TX interrupt\n");
1187                return IRQ_NONE;
1188        }
1189
1190        for (ring = 0; ring < dev->num_tx_queues; ring++) {
1191                if (!(priv->irq1_stat & BIT(ring)))
1192                        continue;
1193
1194                txr = &priv->tx_rings[ring];
1195
1196                if (likely(napi_schedule_prep(&txr->napi))) {
1197                        intrl2_1_mask_set(priv, BIT(ring));
1198                        __napi_schedule_irqoff(&txr->napi);
1199                }
1200        }
1201
1202        return IRQ_HANDLED;
1203}
1204
1205static irqreturn_t bcm_sysport_wol_isr(int irq, void *dev_id)
1206{
1207        struct bcm_sysport_priv *priv = dev_id;
1208
1209        pm_wakeup_event(&priv->pdev->dev, 0);
1210
1211        return IRQ_HANDLED;
1212}
1213
1214#ifdef CONFIG_NET_POLL_CONTROLLER
1215static void bcm_sysport_poll_controller(struct net_device *dev)
1216{
1217        struct bcm_sysport_priv *priv = netdev_priv(dev);
1218
1219        disable_irq(priv->irq0);
1220        bcm_sysport_rx_isr(priv->irq0, priv);
1221        enable_irq(priv->irq0);
1222
1223        if (!priv->is_lite) {
1224                disable_irq(priv->irq1);
1225                bcm_sysport_tx_isr(priv->irq1, priv);
1226                enable_irq(priv->irq1);
1227        }
1228}
1229#endif
1230
1231static struct sk_buff *bcm_sysport_insert_tsb(struct sk_buff *skb,
1232                                              struct net_device *dev)
1233{
1234        struct bcm_sysport_priv *priv = netdev_priv(dev);
1235        struct sk_buff *nskb;
1236        struct bcm_tsb *tsb;
1237        u32 csum_info;
1238        u8 ip_proto;
1239        u16 csum_start;
1240        __be16 ip_ver;
1241
1242        /* Re-allocate SKB if needed */
1243        if (unlikely(skb_headroom(skb) < sizeof(*tsb))) {
1244                nskb = skb_realloc_headroom(skb, sizeof(*tsb));
1245                if (!nskb) {
1246                        dev_kfree_skb_any(skb);
1247                        priv->mib.tx_realloc_tsb_failed++;
1248                        dev->stats.tx_errors++;
1249                        dev->stats.tx_dropped++;
1250                        return NULL;
1251                }
1252                dev_consume_skb_any(skb);
1253                skb = nskb;
1254                priv->mib.tx_realloc_tsb++;
1255        }
1256
1257        tsb = skb_push(skb, sizeof(*tsb));
1258        /* Zero-out TSB by default */
1259        memset(tsb, 0, sizeof(*tsb));
1260
1261        if (skb_vlan_tag_present(skb)) {
1262                tsb->pcp_dei_vid = skb_vlan_tag_get_prio(skb) & PCP_DEI_MASK;
1263                tsb->pcp_dei_vid |= (u32)skb_vlan_tag_get_id(skb) << VID_SHIFT;
1264        }
1265
1266        if (skb->ip_summed == CHECKSUM_PARTIAL) {
1267                ip_ver = skb->protocol;
1268                switch (ip_ver) {
1269                case htons(ETH_P_IP):
1270                        ip_proto = ip_hdr(skb)->protocol;
1271                        break;
1272                case htons(ETH_P_IPV6):
1273                        ip_proto = ipv6_hdr(skb)->nexthdr;
1274                        break;
1275                default:
1276                        return skb;
1277                }
1278
1279                /* Get the checksum offset and the L4 (transport) offset */
1280                csum_start = skb_checksum_start_offset(skb) - sizeof(*tsb);
1281                /* Account for the HW inserted VLAN tag */
1282                if (skb_vlan_tag_present(skb))
1283                        csum_start += VLAN_HLEN;
1284                csum_info = (csum_start + skb->csum_offset) & L4_CSUM_PTR_MASK;
1285                csum_info |= (csum_start << L4_PTR_SHIFT);
1286
1287                if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) {
1288                        csum_info |= L4_LENGTH_VALID;
1289                        if (ip_proto == IPPROTO_UDP &&
1290                            ip_ver == htons(ETH_P_IP))
1291                                csum_info |= L4_UDP;
1292                } else {
1293                        csum_info = 0;
1294                }
1295
1296                tsb->l4_ptr_dest_map = csum_info;
1297        }
1298
1299        return skb;
1300}
1301
1302static netdev_tx_t bcm_sysport_xmit(struct sk_buff *skb,
1303                                    struct net_device *dev)
1304{
1305        struct bcm_sysport_priv *priv = netdev_priv(dev);
1306        struct device *kdev = &priv->pdev->dev;
1307        struct bcm_sysport_tx_ring *ring;
1308        struct bcm_sysport_cb *cb;
1309        struct netdev_queue *txq;
1310        u32 len_status, addr_lo;
1311        unsigned int skb_len;
1312        unsigned long flags;
1313        dma_addr_t mapping;
1314        u16 queue;
1315        int ret;
1316
1317        queue = skb_get_queue_mapping(skb);
1318        txq = netdev_get_tx_queue(dev, queue);
1319        ring = &priv->tx_rings[queue];
1320
1321        /* lock against tx reclaim in BH context and TX ring full interrupt */
1322        spin_lock_irqsave(&ring->lock, flags);
1323        if (unlikely(ring->desc_count == 0)) {
1324                netif_tx_stop_queue(txq);
1325                netdev_err(dev, "queue %d awake and ring full!\n", queue);
1326                ret = NETDEV_TX_BUSY;
1327                goto out;
1328        }
1329
1330        /* Insert TSB and checksum infos */
1331        if (priv->tsb_en) {
1332                skb = bcm_sysport_insert_tsb(skb, dev);
1333                if (!skb) {
1334                        ret = NETDEV_TX_OK;
1335                        goto out;
1336                }
1337        }
1338
1339        skb_len = skb->len;
1340
1341        mapping = dma_map_single(kdev, skb->data, skb_len, DMA_TO_DEVICE);
1342        if (dma_mapping_error(kdev, mapping)) {
1343                priv->mib.tx_dma_failed++;
1344                netif_err(priv, tx_err, dev, "DMA map failed at %p (len=%d)\n",
1345                          skb->data, skb_len);
1346                ret = NETDEV_TX_OK;
1347                goto out;
1348        }
1349
1350        /* Remember the SKB for future freeing */
1351        cb = &ring->cbs[ring->curr_desc];
1352        cb->skb = skb;
1353        dma_unmap_addr_set(cb, dma_addr, mapping);
1354        dma_unmap_len_set(cb, dma_len, skb_len);
1355
1356        addr_lo = lower_32_bits(mapping);
1357        len_status = upper_32_bits(mapping) & DESC_ADDR_HI_MASK;
1358        len_status |= (skb_len << DESC_LEN_SHIFT);
1359        len_status |= (DESC_SOP | DESC_EOP | TX_STATUS_APP_CRC) <<
1360                       DESC_STATUS_SHIFT;
1361        if (skb->ip_summed == CHECKSUM_PARTIAL)
1362                len_status |= (DESC_L4_CSUM << DESC_STATUS_SHIFT);
1363        if (skb_vlan_tag_present(skb))
1364                len_status |= (TX_STATUS_VLAN_VID_TSB << DESC_STATUS_SHIFT);
1365
1366        ring->curr_desc++;
1367        if (ring->curr_desc == ring->size)
1368                ring->curr_desc = 0;
1369        ring->desc_count--;
1370
1371        /* Ports are latched, so write upper address first */
1372        tdma_writel(priv, len_status, TDMA_WRITE_PORT_HI(ring->index));
1373        tdma_writel(priv, addr_lo, TDMA_WRITE_PORT_LO(ring->index));
1374
1375        /* Check ring space and update SW control flow */
1376        if (ring->desc_count == 0)
1377                netif_tx_stop_queue(txq);
1378
1379        netif_dbg(priv, tx_queued, dev, "ring=%d desc_count=%d, curr_desc=%d\n",
1380                  ring->index, ring->desc_count, ring->curr_desc);
1381
1382        ret = NETDEV_TX_OK;
1383out:
1384        spin_unlock_irqrestore(&ring->lock, flags);
1385        return ret;
1386}
1387
1388static void bcm_sysport_tx_timeout(struct net_device *dev, unsigned int txqueue)
1389{
1390        netdev_warn(dev, "transmit timeout!\n");
1391
1392        netif_trans_update(dev);
1393        dev->stats.tx_errors++;
1394
1395        netif_tx_wake_all_queues(dev);
1396}
1397
1398/* phylib adjust link callback */
1399static void bcm_sysport_adj_link(struct net_device *dev)
1400{
1401        struct bcm_sysport_priv *priv = netdev_priv(dev);
1402        struct phy_device *phydev = dev->phydev;
1403        unsigned int changed = 0;
1404        u32 cmd_bits = 0, reg;
1405
1406        if (priv->old_link != phydev->link) {
1407                changed = 1;
1408                priv->old_link = phydev->link;
1409        }
1410
1411        if (priv->old_duplex != phydev->duplex) {
1412                changed = 1;
1413                priv->old_duplex = phydev->duplex;
1414        }
1415
1416        if (priv->is_lite)
1417                goto out;
1418
1419        switch (phydev->speed) {
1420        case SPEED_2500:
1421                cmd_bits = CMD_SPEED_2500;
1422                break;
1423        case SPEED_1000:
1424                cmd_bits = CMD_SPEED_1000;
1425                break;
1426        case SPEED_100:
1427                cmd_bits = CMD_SPEED_100;
1428                break;
1429        case SPEED_10:
1430                cmd_bits = CMD_SPEED_10;
1431                break;
1432        default:
1433                break;
1434        }
1435        cmd_bits <<= CMD_SPEED_SHIFT;
1436
1437        if (phydev->duplex == DUPLEX_HALF)
1438                cmd_bits |= CMD_HD_EN;
1439
1440        if (priv->old_pause != phydev->pause) {
1441                changed = 1;
1442                priv->old_pause = phydev->pause;
1443        }
1444
1445        if (!phydev->pause)
1446                cmd_bits |= CMD_RX_PAUSE_IGNORE | CMD_TX_PAUSE_IGNORE;
1447
1448        if (!changed)
1449                return;
1450
1451        if (phydev->link) {
1452                reg = umac_readl(priv, UMAC_CMD);
1453                reg &= ~((CMD_SPEED_MASK << CMD_SPEED_SHIFT) |
1454                        CMD_HD_EN | CMD_RX_PAUSE_IGNORE |
1455                        CMD_TX_PAUSE_IGNORE);
1456                reg |= cmd_bits;
1457                umac_writel(priv, reg, UMAC_CMD);
1458        }
1459out:
1460        if (changed)
1461                phy_print_status(phydev);
1462}
1463
1464static void bcm_sysport_init_dim(struct bcm_sysport_priv *priv,
1465                                 void (*cb)(struct work_struct *work))
1466{
1467        struct bcm_sysport_net_dim *dim = &priv->dim;
1468
1469        INIT_WORK(&dim->dim.work, cb);
1470        dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1471        dim->event_ctr = 0;
1472        dim->packets = 0;
1473        dim->bytes = 0;
1474}
1475
1476static void bcm_sysport_init_rx_coalesce(struct bcm_sysport_priv *priv)
1477{
1478        struct bcm_sysport_net_dim *dim = &priv->dim;
1479        struct dim_cq_moder moder;
1480        u32 usecs, pkts;
1481
1482        usecs = priv->rx_coalesce_usecs;
1483        pkts = priv->rx_max_coalesced_frames;
1484
1485        /* If DIM was enabled, re-apply default parameters */
1486        if (dim->use_dim) {
1487                moder = net_dim_get_def_rx_moderation(dim->dim.mode);
1488                usecs = moder.usec;
1489                pkts = moder.pkts;
1490        }
1491
1492        bcm_sysport_set_rx_coalesce(priv, usecs, pkts);
1493}
1494
1495static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv,
1496                                    unsigned int index)
1497{
1498        struct bcm_sysport_tx_ring *ring = &priv->tx_rings[index];
1499        size_t size;
1500        u32 reg;
1501
1502        /* Simple descriptors partitioning for now */
1503        size = 256;
1504
1505        ring->cbs = kcalloc(size, sizeof(struct bcm_sysport_cb), GFP_KERNEL);
1506        if (!ring->cbs) {
1507                netif_err(priv, hw, priv->netdev, "CB allocation failed\n");
1508                return -ENOMEM;
1509        }
1510
1511        /* Initialize SW view of the ring */
1512        spin_lock_init(&ring->lock);
1513        ring->priv = priv;
1514        netif_tx_napi_add(priv->netdev, &ring->napi, bcm_sysport_tx_poll, 64);
1515        ring->index = index;
1516        ring->size = size;
1517        ring->clean_index = 0;
1518        ring->alloc_size = ring->size;
1519        ring->desc_count = ring->size;
1520        ring->curr_desc = 0;
1521
1522        /* Initialize HW ring */
1523        tdma_writel(priv, RING_EN, TDMA_DESC_RING_HEAD_TAIL_PTR(index));
1524        tdma_writel(priv, 0, TDMA_DESC_RING_COUNT(index));
1525        tdma_writel(priv, 1, TDMA_DESC_RING_INTR_CONTROL(index));
1526        tdma_writel(priv, 0, TDMA_DESC_RING_PROD_CONS_INDEX(index));
1527
1528        /* Configure QID and port mapping */
1529        reg = tdma_readl(priv, TDMA_DESC_RING_MAPPING(index));
1530        reg &= ~(RING_QID_MASK | RING_PORT_ID_MASK << RING_PORT_ID_SHIFT);
1531        if (ring->inspect) {
1532                reg |= ring->switch_queue & RING_QID_MASK;
1533                reg |= ring->switch_port << RING_PORT_ID_SHIFT;
1534        } else {
1535                reg |= RING_IGNORE_STATUS;
1536        }
1537        tdma_writel(priv, reg, TDMA_DESC_RING_MAPPING(index));
1538        reg = 0;
1539        /* Adjust the packet size calculations if SYSTEMPORT is responsible
1540         * for HW insertion of VLAN tags
1541         */
1542        if (priv->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)
1543                reg = VLAN_HLEN << RING_PKT_SIZE_ADJ_SHIFT;
1544        tdma_writel(priv, reg, TDMA_DESC_RING_PCP_DEI_VID(index));
1545
1546        /* Enable ACB algorithm 2 */
1547        reg = tdma_readl(priv, TDMA_CONTROL);
1548        reg |= tdma_control_bit(priv, ACB_ALGO);
1549        tdma_writel(priv, reg, TDMA_CONTROL);
1550
1551        /* Do not use tdma_control_bit() here because TSB_SWAP1 collides
1552         * with the original definition of ACB_ALGO
1553         */
1554        reg = tdma_readl(priv, TDMA_CONTROL);
1555        if (priv->is_lite)
1556                reg &= ~BIT(TSB_SWAP1);
1557        /* Set a correct TSB format based on host endian */
1558        if (!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1559                reg |= tdma_control_bit(priv, TSB_SWAP0);
1560        else
1561                reg &= ~tdma_control_bit(priv, TSB_SWAP0);
1562        tdma_writel(priv, reg, TDMA_CONTROL);
1563
1564        /* Program the number of descriptors as MAX_THRESHOLD and half of
1565         * its size for the hysteresis trigger
1566         */
1567        tdma_writel(priv, ring->size |
1568                        1 << RING_HYST_THRESH_SHIFT,
1569                        TDMA_DESC_RING_MAX_HYST(index));
1570
1571        /* Enable the ring queue in the arbiter */
1572        reg = tdma_readl(priv, TDMA_TIER1_ARB_0_QUEUE_EN);
1573        reg |= (1 << index);
1574        tdma_writel(priv, reg, TDMA_TIER1_ARB_0_QUEUE_EN);
1575
1576        napi_enable(&ring->napi);
1577
1578        netif_dbg(priv, hw, priv->netdev,
1579                  "TDMA cfg, size=%d, switch q=%d,port=%d\n",
1580                  ring->size, ring->switch_queue,
1581                  ring->switch_port);
1582
1583        return 0;
1584}
1585
1586static void bcm_sysport_fini_tx_ring(struct bcm_sysport_priv *priv,
1587                                     unsigned int index)
1588{
1589        struct bcm_sysport_tx_ring *ring = &priv->tx_rings[index];
1590        u32 reg;
1591
1592        /* Caller should stop the TDMA engine */
1593        reg = tdma_readl(priv, TDMA_STATUS);
1594        if (!(reg & TDMA_DISABLED))
1595                netdev_warn(priv->netdev, "TDMA not stopped!\n");
1596
1597        /* ring->cbs is the last part in bcm_sysport_init_tx_ring which could
1598         * fail, so by checking this pointer we know whether the TX ring was
1599         * fully initialized or not.
1600         */
1601        if (!ring->cbs)
1602                return;
1603
1604        napi_disable(&ring->napi);
1605        netif_napi_del(&ring->napi);
1606
1607        bcm_sysport_tx_clean(priv, ring);
1608
1609        kfree(ring->cbs);
1610        ring->cbs = NULL;
1611        ring->size = 0;
1612        ring->alloc_size = 0;
1613
1614        netif_dbg(priv, hw, priv->netdev, "TDMA fini done\n");
1615}
1616
1617/* RDMA helper */
1618static inline int rdma_enable_set(struct bcm_sysport_priv *priv,
1619                                  unsigned int enable)
1620{
1621        unsigned int timeout = 1000;
1622        u32 reg;
1623
1624        reg = rdma_readl(priv, RDMA_CONTROL);
1625        if (enable)
1626                reg |= RDMA_EN;
1627        else
1628                reg &= ~RDMA_EN;
1629        rdma_writel(priv, reg, RDMA_CONTROL);
1630
1631        /* Poll for RMDA disabling completion */
1632        do {
1633                reg = rdma_readl(priv, RDMA_STATUS);
1634                if (!!(reg & RDMA_DISABLED) == !enable)
1635                        return 0;
1636                usleep_range(1000, 2000);
1637        } while (timeout-- > 0);
1638
1639        netdev_err(priv->netdev, "timeout waiting for RDMA to finish\n");
1640
1641        return -ETIMEDOUT;
1642}
1643
1644/* TDMA helper */
1645static inline int tdma_enable_set(struct bcm_sysport_priv *priv,
1646                                  unsigned int enable)
1647{
1648        unsigned int timeout = 1000;
1649        u32 reg;
1650
1651        reg = tdma_readl(priv, TDMA_CONTROL);
1652        if (enable)
1653                reg |= tdma_control_bit(priv, TDMA_EN);
1654        else
1655                reg &= ~tdma_control_bit(priv, TDMA_EN);
1656        tdma_writel(priv, reg, TDMA_CONTROL);
1657
1658        /* Poll for TMDA disabling completion */
1659        do {
1660                reg = tdma_readl(priv, TDMA_STATUS);
1661                if (!!(reg & TDMA_DISABLED) == !enable)
1662                        return 0;
1663
1664                usleep_range(1000, 2000);
1665        } while (timeout-- > 0);
1666
1667        netdev_err(priv->netdev, "timeout waiting for TDMA to finish\n");
1668
1669        return -ETIMEDOUT;
1670}
1671
1672static int bcm_sysport_init_rx_ring(struct bcm_sysport_priv *priv)
1673{
1674        struct bcm_sysport_cb *cb;
1675        u32 reg;
1676        int ret;
1677        int i;
1678
1679        /* Initialize SW view of the RX ring */
1680        priv->num_rx_bds = priv->num_rx_desc_words / WORDS_PER_DESC;
1681        priv->rx_bds = priv->base + SYS_PORT_RDMA_OFFSET;
1682        priv->rx_c_index = 0;
1683        priv->rx_read_ptr = 0;
1684        priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct bcm_sysport_cb),
1685                                GFP_KERNEL);
1686        if (!priv->rx_cbs) {
1687                netif_err(priv, hw, priv->netdev, "CB allocation failed\n");
1688                return -ENOMEM;
1689        }
1690
1691        for (i = 0; i < priv->num_rx_bds; i++) {
1692                cb = priv->rx_cbs + i;
1693                cb->bd_addr = priv->rx_bds + i * DESC_SIZE;
1694        }
1695
1696        ret = bcm_sysport_alloc_rx_bufs(priv);
1697        if (ret) {
1698                netif_err(priv, hw, priv->netdev, "SKB allocation failed\n");
1699                return ret;
1700        }
1701
1702        /* Initialize HW, ensure RDMA is disabled */
1703        reg = rdma_readl(priv, RDMA_STATUS);
1704        if (!(reg & RDMA_DISABLED))
1705                rdma_enable_set(priv, 0);
1706
1707        rdma_writel(priv, 0, RDMA_WRITE_PTR_LO);
1708        rdma_writel(priv, 0, RDMA_WRITE_PTR_HI);
1709        rdma_writel(priv, 0, RDMA_PROD_INDEX);
1710        rdma_writel(priv, 0, RDMA_CONS_INDEX);
1711        rdma_writel(priv, priv->num_rx_bds << RDMA_RING_SIZE_SHIFT |
1712                          RX_BUF_LENGTH, RDMA_RING_BUF_SIZE);
1713        /* Operate the queue in ring mode */
1714        rdma_writel(priv, 0, RDMA_START_ADDR_HI);
1715        rdma_writel(priv, 0, RDMA_START_ADDR_LO);
1716        rdma_writel(priv, 0, RDMA_END_ADDR_HI);
1717        rdma_writel(priv, priv->num_rx_desc_words - 1, RDMA_END_ADDR_LO);
1718
1719        netif_dbg(priv, hw, priv->netdev,
1720                  "RDMA cfg, num_rx_bds=%d, rx_bds=%p\n",
1721                  priv->num_rx_bds, priv->rx_bds);
1722
1723        return 0;
1724}
1725
1726static void bcm_sysport_fini_rx_ring(struct bcm_sysport_priv *priv)
1727{
1728        struct bcm_sysport_cb *cb;
1729        unsigned int i;
1730        u32 reg;
1731
1732        /* Caller should ensure RDMA is disabled */
1733        reg = rdma_readl(priv, RDMA_STATUS);
1734        if (!(reg & RDMA_DISABLED))
1735                netdev_warn(priv->netdev, "RDMA not stopped!\n");
1736
1737        for (i = 0; i < priv->num_rx_bds; i++) {
1738                cb = &priv->rx_cbs[i];
1739                if (dma_unmap_addr(cb, dma_addr))
1740                        dma_unmap_single(&priv->pdev->dev,
1741                                         dma_unmap_addr(cb, dma_addr),
1742                                         RX_BUF_LENGTH, DMA_FROM_DEVICE);
1743                bcm_sysport_free_cb(cb);
1744        }
1745
1746        kfree(priv->rx_cbs);
1747        priv->rx_cbs = NULL;
1748
1749        netif_dbg(priv, hw, priv->netdev, "RDMA fini done\n");
1750}
1751
1752static void bcm_sysport_set_rx_mode(struct net_device *dev)
1753{
1754        struct bcm_sysport_priv *priv = netdev_priv(dev);
1755        u32 reg;
1756
1757        if (priv->is_lite)
1758                return;
1759
1760        reg = umac_readl(priv, UMAC_CMD);
1761        if (dev->flags & IFF_PROMISC)
1762                reg |= CMD_PROMISC;
1763        else
1764                reg &= ~CMD_PROMISC;
1765        umac_writel(priv, reg, UMAC_CMD);
1766
1767        /* No support for ALLMULTI */
1768        if (dev->flags & IFF_ALLMULTI)
1769                return;
1770}
1771
1772static inline void umac_enable_set(struct bcm_sysport_priv *priv,
1773                                   u32 mask, unsigned int enable)
1774{
1775        u32 reg;
1776
1777        if (!priv->is_lite) {
1778                reg = umac_readl(priv, UMAC_CMD);
1779                if (enable)
1780                        reg |= mask;
1781                else
1782                        reg &= ~mask;
1783                umac_writel(priv, reg, UMAC_CMD);
1784        } else {
1785                reg = gib_readl(priv, GIB_CONTROL);
1786                if (enable)
1787                        reg |= mask;
1788                else
1789                        reg &= ~mask;
1790                gib_writel(priv, reg, GIB_CONTROL);
1791        }
1792
1793        /* UniMAC stops on a packet boundary, wait for a full-sized packet
1794         * to be processed (1 msec).
1795         */
1796        if (enable == 0)
1797                usleep_range(1000, 2000);
1798}
1799
1800static inline void umac_reset(struct bcm_sysport_priv *priv)
1801{
1802        u32 reg;
1803
1804        if (priv->is_lite)
1805                return;
1806
1807        reg = umac_readl(priv, UMAC_CMD);
1808        reg |= CMD_SW_RESET;
1809        umac_writel(priv, reg, UMAC_CMD);
1810        udelay(10);
1811        reg = umac_readl(priv, UMAC_CMD);
1812        reg &= ~CMD_SW_RESET;
1813        umac_writel(priv, reg, UMAC_CMD);
1814}
1815
1816static void umac_set_hw_addr(struct bcm_sysport_priv *priv,
1817                             unsigned char *addr)
1818{
1819        u32 mac0 = (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) |
1820                    addr[3];
1821        u32 mac1 = (addr[4] << 8) | addr[5];
1822
1823        if (!priv->is_lite) {
1824                umac_writel(priv, mac0, UMAC_MAC0);
1825                umac_writel(priv, mac1, UMAC_MAC1);
1826        } else {
1827                gib_writel(priv, mac0, GIB_MAC0);
1828                gib_writel(priv, mac1, GIB_MAC1);
1829        }
1830}
1831
1832static void topctrl_flush(struct bcm_sysport_priv *priv)
1833{
1834        topctrl_writel(priv, RX_FLUSH, RX_FLUSH_CNTL);
1835        topctrl_writel(priv, TX_FLUSH, TX_FLUSH_CNTL);
1836        mdelay(1);
1837        topctrl_writel(priv, 0, RX_FLUSH_CNTL);
1838        topctrl_writel(priv, 0, TX_FLUSH_CNTL);
1839}
1840
1841static int bcm_sysport_change_mac(struct net_device *dev, void *p)
1842{
1843        struct bcm_sysport_priv *priv = netdev_priv(dev);
1844        struct sockaddr *addr = p;
1845
1846        if (!is_valid_ether_addr(addr->sa_data))
1847                return -EINVAL;
1848
1849        memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1850
1851        /* interface is disabled, changes to MAC will be reflected on next
1852         * open call
1853         */
1854        if (!netif_running(dev))
1855                return 0;
1856
1857        umac_set_hw_addr(priv, dev->dev_addr);
1858
1859        return 0;
1860}
1861
1862static void bcm_sysport_get_stats64(struct net_device *dev,
1863                                    struct rtnl_link_stats64 *stats)
1864{
1865        struct bcm_sysport_priv *priv = netdev_priv(dev);
1866        struct bcm_sysport_stats64 *stats64 = &priv->stats64;
1867        unsigned int start;
1868
1869        netdev_stats_to_stats64(stats, &dev->stats);
1870
1871        bcm_sysport_update_tx_stats(priv, &stats->tx_bytes,
1872                                    &stats->tx_packets);
1873
1874        do {
1875                start = u64_stats_fetch_begin_irq(&priv->syncp);
1876                stats->rx_packets = stats64->rx_packets;
1877                stats->rx_bytes = stats64->rx_bytes;
1878        } while (u64_stats_fetch_retry_irq(&priv->syncp, start));
1879}
1880
1881static void bcm_sysport_netif_start(struct net_device *dev)
1882{
1883        struct bcm_sysport_priv *priv = netdev_priv(dev);
1884
1885        /* Enable NAPI */
1886        bcm_sysport_init_dim(priv, bcm_sysport_dim_work);
1887        bcm_sysport_init_rx_coalesce(priv);
1888        napi_enable(&priv->napi);
1889
1890        /* Enable RX interrupt and TX ring full interrupt */
1891        intrl2_0_mask_clear(priv, INTRL2_0_RDMA_MBDONE | INTRL2_0_TX_RING_FULL);
1892
1893        phy_start(dev->phydev);
1894
1895        /* Enable TX interrupts for the TXQs */
1896        if (!priv->is_lite)
1897                intrl2_1_mask_clear(priv, 0xffffffff);
1898        else
1899                intrl2_0_mask_clear(priv, INTRL2_0_TDMA_MBDONE_MASK);
1900}
1901
1902static void rbuf_init(struct bcm_sysport_priv *priv)
1903{
1904        u32 reg;
1905
1906        reg = rbuf_readl(priv, RBUF_CONTROL);
1907        reg |= RBUF_4B_ALGN | RBUF_RSB_EN;
1908        /* Set a correct RSB format on SYSTEMPORT Lite */
1909        if (priv->is_lite)
1910                reg &= ~RBUF_RSB_SWAP1;
1911
1912        /* Set a correct RSB format based on host endian */
1913        if (!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1914                reg |= RBUF_RSB_SWAP0;
1915        else
1916                reg &= ~RBUF_RSB_SWAP0;
1917        rbuf_writel(priv, reg, RBUF_CONTROL);
1918}
1919
1920static inline void bcm_sysport_mask_all_intrs(struct bcm_sysport_priv *priv)
1921{
1922        intrl2_0_mask_set(priv, 0xffffffff);
1923        intrl2_0_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
1924        if (!priv->is_lite) {
1925                intrl2_1_mask_set(priv, 0xffffffff);
1926                intrl2_1_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
1927        }
1928}
1929
1930static inline void gib_set_pad_extension(struct bcm_sysport_priv *priv)
1931{
1932        u32 reg;
1933
1934        reg = gib_readl(priv, GIB_CONTROL);
1935        /* Include Broadcom tag in pad extension and fix up IPG_LENGTH */
1936        if (netdev_uses_dsa(priv->netdev)) {
1937                reg &= ~(GIB_PAD_EXTENSION_MASK << GIB_PAD_EXTENSION_SHIFT);
1938                reg |= ENET_BRCM_TAG_LEN << GIB_PAD_EXTENSION_SHIFT;
1939        }
1940        reg &= ~(GIB_IPG_LEN_MASK << GIB_IPG_LEN_SHIFT);
1941        reg |= 12 << GIB_IPG_LEN_SHIFT;
1942        gib_writel(priv, reg, GIB_CONTROL);
1943}
1944
1945static int bcm_sysport_open(struct net_device *dev)
1946{
1947        struct bcm_sysport_priv *priv = netdev_priv(dev);
1948        struct phy_device *phydev;
1949        unsigned int i;
1950        int ret;
1951
1952        clk_prepare_enable(priv->clk);
1953
1954        /* Reset UniMAC */
1955        umac_reset(priv);
1956
1957        /* Flush TX and RX FIFOs at TOPCTRL level */
1958        topctrl_flush(priv);
1959
1960        /* Disable the UniMAC RX/TX */
1961        umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, 0);
1962
1963        /* Enable RBUF 2bytes alignment and Receive Status Block */
1964        rbuf_init(priv);
1965
1966        /* Set maximum frame length */
1967        if (!priv->is_lite)
1968                umac_writel(priv, UMAC_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
1969        else
1970                gib_set_pad_extension(priv);
1971
1972        /* Apply features again in case we changed them while interface was
1973         * down
1974         */
1975        bcm_sysport_set_features(dev, dev->features);
1976
1977        /* Set MAC address */
1978        umac_set_hw_addr(priv, dev->dev_addr);
1979
1980        phydev = of_phy_connect(dev, priv->phy_dn, bcm_sysport_adj_link,
1981                                0, priv->phy_interface);
1982        if (!phydev) {
1983                netdev_err(dev, "could not attach to PHY\n");
1984                ret = -ENODEV;
1985                goto out_clk_disable;
1986        }
1987
1988        /* Reset house keeping link status */
1989        priv->old_duplex = -1;
1990        priv->old_link = -1;
1991        priv->old_pause = -1;
1992
1993        /* mask all interrupts and request them */
1994        bcm_sysport_mask_all_intrs(priv);
1995
1996        ret = request_irq(priv->irq0, bcm_sysport_rx_isr, 0, dev->name, dev);
1997        if (ret) {
1998                netdev_err(dev, "failed to request RX interrupt\n");
1999                goto out_phy_disconnect;
2000        }
2001
2002        if (!priv->is_lite) {
2003                ret = request_irq(priv->irq1, bcm_sysport_tx_isr, 0,
2004                                  dev->name, dev);
2005                if (ret) {
2006                        netdev_err(dev, "failed to request TX interrupt\n");
2007                        goto out_free_irq0;
2008                }
2009        }
2010
2011        /* Initialize both hardware and software ring */
2012        for (i = 0; i < dev->num_tx_queues; i++) {
2013                ret = bcm_sysport_init_tx_ring(priv, i);
2014                if (ret) {
2015                        netdev_err(dev, "failed to initialize TX ring %d\n",
2016                                   i);
2017                        goto out_free_tx_ring;
2018                }
2019        }
2020
2021        /* Initialize linked-list */
2022        tdma_writel(priv, TDMA_LL_RAM_INIT_BUSY, TDMA_STATUS);
2023
2024        /* Initialize RX ring */
2025        ret = bcm_sysport_init_rx_ring(priv);
2026        if (ret) {
2027                netdev_err(dev, "failed to initialize RX ring\n");
2028                goto out_free_rx_ring;
2029        }
2030
2031        /* Turn on RDMA */
2032        ret = rdma_enable_set(priv, 1);
2033        if (ret)
2034                goto out_free_rx_ring;
2035
2036        /* Turn on TDMA */
2037        ret = tdma_enable_set(priv, 1);
2038        if (ret)
2039                goto out_clear_rx_int;
2040
2041        /* Turn on UniMAC TX/RX */
2042        umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, 1);
2043
2044        bcm_sysport_netif_start(dev);
2045
2046        netif_tx_start_all_queues(dev);
2047
2048        return 0;
2049
2050out_clear_rx_int:
2051        intrl2_0_mask_set(priv, INTRL2_0_RDMA_MBDONE | INTRL2_0_TX_RING_FULL);
2052out_free_rx_ring:
2053        bcm_sysport_fini_rx_ring(priv);
2054out_free_tx_ring:
2055        for (i = 0; i < dev->num_tx_queues; i++)
2056                bcm_sysport_fini_tx_ring(priv, i);
2057        if (!priv->is_lite)
2058                free_irq(priv->irq1, dev);
2059out_free_irq0:
2060        free_irq(priv->irq0, dev);
2061out_phy_disconnect:
2062        phy_disconnect(phydev);
2063out_clk_disable:
2064        clk_disable_unprepare(priv->clk);
2065        return ret;
2066}
2067
2068static void bcm_sysport_netif_stop(struct net_device *dev)
2069{
2070        struct bcm_sysport_priv *priv = netdev_priv(dev);
2071
2072        /* stop all software from updating hardware */
2073        netif_tx_disable(dev);
2074        napi_disable(&priv->napi);
2075        cancel_work_sync(&priv->dim.dim.work);
2076        phy_stop(dev->phydev);
2077
2078        /* mask all interrupts */
2079        bcm_sysport_mask_all_intrs(priv);
2080}
2081
2082static int bcm_sysport_stop(struct net_device *dev)
2083{
2084        struct bcm_sysport_priv *priv = netdev_priv(dev);
2085        unsigned int i;
2086        int ret;
2087
2088        bcm_sysport_netif_stop(dev);
2089
2090        /* Disable UniMAC RX */
2091        umac_enable_set(priv, CMD_RX_EN, 0);
2092
2093        ret = tdma_enable_set(priv, 0);
2094        if (ret) {
2095                netdev_err(dev, "timeout disabling RDMA\n");
2096                return ret;
2097        }
2098
2099        /* Wait for a maximum packet size to be drained */
2100        usleep_range(2000, 3000);
2101
2102        ret = rdma_enable_set(priv, 0);
2103        if (ret) {
2104                netdev_err(dev, "timeout disabling TDMA\n");
2105                return ret;
2106        }
2107
2108        /* Disable UniMAC TX */
2109        umac_enable_set(priv, CMD_TX_EN, 0);
2110
2111        /* Free RX/TX rings SW structures */
2112        for (i = 0; i < dev->num_tx_queues; i++)
2113                bcm_sysport_fini_tx_ring(priv, i);
2114        bcm_sysport_fini_rx_ring(priv);
2115
2116        free_irq(priv->irq0, dev);
2117        if (!priv->is_lite)
2118                free_irq(priv->irq1, dev);
2119
2120        /* Disconnect from PHY */
2121        phy_disconnect(dev->phydev);
2122
2123        clk_disable_unprepare(priv->clk);
2124
2125        return 0;
2126}
2127
2128static int bcm_sysport_rule_find(struct bcm_sysport_priv *priv,
2129                                 u64 location)
2130{
2131        unsigned int index;
2132        u32 reg;
2133
2134        for_each_set_bit(index, priv->filters, RXCHK_BRCM_TAG_MAX) {
2135                reg = rxchk_readl(priv, RXCHK_BRCM_TAG(index));
2136                reg >>= RXCHK_BRCM_TAG_CID_SHIFT;
2137                reg &= RXCHK_BRCM_TAG_CID_MASK;
2138                if (reg == location)
2139                        return index;
2140        }
2141
2142        return -EINVAL;
2143}
2144
2145static int bcm_sysport_rule_get(struct bcm_sysport_priv *priv,
2146                                struct ethtool_rxnfc *nfc)
2147{
2148        int index;
2149
2150        /* This is not a rule that we know about */
2151        index = bcm_sysport_rule_find(priv, nfc->fs.location);
2152        if (index < 0)
2153                return -EOPNOTSUPP;
2154
2155        nfc->fs.ring_cookie = RX_CLS_FLOW_WAKE;
2156
2157        return 0;
2158}
2159
2160static int bcm_sysport_rule_set(struct bcm_sysport_priv *priv,
2161                                struct ethtool_rxnfc *nfc)
2162{
2163        unsigned int index;
2164        u32 reg;
2165
2166        /* We cannot match locations greater than what the classification ID
2167         * permits (256 entries)
2168         */
2169        if (nfc->fs.location > RXCHK_BRCM_TAG_CID_MASK)
2170                return -E2BIG;
2171
2172        /* We cannot support flows that are not destined for a wake-up */
2173        if (nfc->fs.ring_cookie != RX_CLS_FLOW_WAKE)
2174                return -EOPNOTSUPP;
2175
2176        /* All filters are already in use, we cannot match more rules */
2177        if (bitmap_weight(priv->filters, RXCHK_BRCM_TAG_MAX) ==
2178            RXCHK_BRCM_TAG_MAX)
2179                return -ENOSPC;
2180
2181        index = find_first_zero_bit(priv->filters, RXCHK_BRCM_TAG_MAX);
2182        if (index >= RXCHK_BRCM_TAG_MAX)
2183                return -ENOSPC;
2184
2185        /* Location is the classification ID, and index is the position
2186         * within one of our 8 possible filters to be programmed
2187         */
2188        reg = rxchk_readl(priv, RXCHK_BRCM_TAG(index));
2189        reg &= ~(RXCHK_BRCM_TAG_CID_MASK << RXCHK_BRCM_TAG_CID_SHIFT);
2190        reg |= nfc->fs.location << RXCHK_BRCM_TAG_CID_SHIFT;
2191        rxchk_writel(priv, reg, RXCHK_BRCM_TAG(index));
2192        rxchk_writel(priv, 0xff00ffff, RXCHK_BRCM_TAG_MASK(index));
2193
2194        priv->filters_loc[index] = nfc->fs.location;
2195        set_bit(index, priv->filters);
2196
2197        return 0;
2198}
2199
2200static int bcm_sysport_rule_del(struct bcm_sysport_priv *priv,
2201                                u64 location)
2202{
2203        int index;
2204
2205        /* This is not a rule that we know about */
2206        index = bcm_sysport_rule_find(priv, location);
2207        if (index < 0)
2208                return -EOPNOTSUPP;
2209
2210        /* No need to disable this filter if it was enabled, this will
2211         * be taken care of during suspend time by bcm_sysport_suspend_to_wol
2212         */
2213        clear_bit(index, priv->filters);
2214        priv->filters_loc[index] = 0;
2215
2216        return 0;
2217}
2218
2219static int bcm_sysport_get_rxnfc(struct net_device *dev,
2220                                 struct ethtool_rxnfc *nfc, u32 *rule_locs)
2221{
2222        struct bcm_sysport_priv *priv = netdev_priv(dev);
2223        int ret = -EOPNOTSUPP;
2224
2225        switch (nfc->cmd) {
2226        case ETHTOOL_GRXCLSRULE:
2227                ret = bcm_sysport_rule_get(priv, nfc);
2228                break;
2229        default:
2230                break;
2231        }
2232
2233        return ret;
2234}
2235
2236static int bcm_sysport_set_rxnfc(struct net_device *dev,
2237                                 struct ethtool_rxnfc *nfc)
2238{
2239        struct bcm_sysport_priv *priv = netdev_priv(dev);
2240        int ret = -EOPNOTSUPP;
2241
2242        switch (nfc->cmd) {
2243        case ETHTOOL_SRXCLSRLINS:
2244                ret = bcm_sysport_rule_set(priv, nfc);
2245                break;
2246        case ETHTOOL_SRXCLSRLDEL:
2247                ret = bcm_sysport_rule_del(priv, nfc->fs.location);
2248                break;
2249        default:
2250                break;
2251        }
2252
2253        return ret;
2254}
2255
2256static const struct ethtool_ops bcm_sysport_ethtool_ops = {
2257        .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
2258                                     ETHTOOL_COALESCE_MAX_FRAMES |
2259                                     ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
2260        .get_drvinfo            = bcm_sysport_get_drvinfo,
2261        .get_msglevel           = bcm_sysport_get_msglvl,
2262        .set_msglevel           = bcm_sysport_set_msglvl,
2263        .get_link               = ethtool_op_get_link,
2264        .get_strings            = bcm_sysport_get_strings,
2265        .get_ethtool_stats      = bcm_sysport_get_stats,
2266        .get_sset_count         = bcm_sysport_get_sset_count,
2267        .get_wol                = bcm_sysport_get_wol,
2268        .set_wol                = bcm_sysport_set_wol,
2269        .get_coalesce           = bcm_sysport_get_coalesce,
2270        .set_coalesce           = bcm_sysport_set_coalesce,
2271        .get_link_ksettings     = phy_ethtool_get_link_ksettings,
2272        .set_link_ksettings     = phy_ethtool_set_link_ksettings,
2273        .get_rxnfc              = bcm_sysport_get_rxnfc,
2274        .set_rxnfc              = bcm_sysport_set_rxnfc,
2275};
2276
2277static u16 bcm_sysport_select_queue(struct net_device *dev, struct sk_buff *skb,
2278                                    struct net_device *sb_dev)
2279{
2280        struct bcm_sysport_priv *priv = netdev_priv(dev);
2281        u16 queue = skb_get_queue_mapping(skb);
2282        struct bcm_sysport_tx_ring *tx_ring;
2283        unsigned int q, port;
2284
2285        if (!netdev_uses_dsa(dev))
2286                return netdev_pick_tx(dev, skb, NULL);
2287
2288        /* DSA tagging layer will have configured the correct queue */
2289        q = BRCM_TAG_GET_QUEUE(queue);
2290        port = BRCM_TAG_GET_PORT(queue);
2291        tx_ring = priv->ring_map[q + port * priv->per_port_num_tx_queues];
2292
2293        if (unlikely(!tx_ring))
2294                return netdev_pick_tx(dev, skb, NULL);
2295
2296        return tx_ring->index;
2297}
2298
2299static const struct net_device_ops bcm_sysport_netdev_ops = {
2300        .ndo_start_xmit         = bcm_sysport_xmit,
2301        .ndo_tx_timeout         = bcm_sysport_tx_timeout,
2302        .ndo_open               = bcm_sysport_open,
2303        .ndo_stop               = bcm_sysport_stop,
2304        .ndo_set_features       = bcm_sysport_set_features,
2305        .ndo_set_rx_mode        = bcm_sysport_set_rx_mode,
2306        .ndo_set_mac_address    = bcm_sysport_change_mac,
2307#ifdef CONFIG_NET_POLL_CONTROLLER
2308        .ndo_poll_controller    = bcm_sysport_poll_controller,
2309#endif
2310        .ndo_get_stats64        = bcm_sysport_get_stats64,
2311        .ndo_select_queue       = bcm_sysport_select_queue,
2312};
2313
2314static int bcm_sysport_map_queues(struct net_device *dev,
2315                                  struct net_device *slave_dev)
2316{
2317        struct dsa_port *dp = dsa_port_from_netdev(slave_dev);
2318        struct bcm_sysport_priv *priv = netdev_priv(dev);
2319        struct bcm_sysport_tx_ring *ring;
2320        unsigned int num_tx_queues;
2321        unsigned int q, qp, port;
2322
2323        /* We can't be setting up queue inspection for non directly attached
2324         * switches
2325         */
2326        if (dp->ds->index)
2327                return 0;
2328
2329        port = dp->index;
2330
2331        /* On SYSTEMPORT Lite we have twice as less queues, so we cannot do a
2332         * 1:1 mapping, we can only do a 2:1 mapping. By reducing the number of
2333         * per-port (slave_dev) network devices queue, we achieve just that.
2334         * This need to happen now before any slave network device is used such
2335         * it accurately reflects the number of real TX queues.
2336         */
2337        if (priv->is_lite)
2338                netif_set_real_num_tx_queues(slave_dev,
2339                                             slave_dev->num_tx_queues / 2);
2340
2341        num_tx_queues = slave_dev->real_num_tx_queues;
2342
2343        if (priv->per_port_num_tx_queues &&
2344            priv->per_port_num_tx_queues != num_tx_queues)
2345                netdev_warn(slave_dev, "asymmetric number of per-port queues\n");
2346
2347        priv->per_port_num_tx_queues = num_tx_queues;
2348
2349        for (q = 0, qp = 0; q < dev->num_tx_queues && qp < num_tx_queues;
2350             q++) {
2351                ring = &priv->tx_rings[q];
2352
2353                if (ring->inspect)
2354                        continue;
2355
2356                /* Just remember the mapping actual programming done
2357                 * during bcm_sysport_init_tx_ring
2358                 */
2359                ring->switch_queue = qp;
2360                ring->switch_port = port;
2361                ring->inspect = true;
2362                priv->ring_map[qp + port * num_tx_queues] = ring;
2363                qp++;
2364        }
2365
2366        return 0;
2367}
2368
2369static int bcm_sysport_unmap_queues(struct net_device *dev,
2370                                    struct net_device *slave_dev)
2371{
2372        struct dsa_port *dp = dsa_port_from_netdev(slave_dev);
2373        struct bcm_sysport_priv *priv = netdev_priv(dev);
2374        struct bcm_sysport_tx_ring *ring;
2375        unsigned int num_tx_queues;
2376        unsigned int q, qp, port;
2377
2378        port = dp->index;
2379
2380        num_tx_queues = slave_dev->real_num_tx_queues;
2381
2382        for (q = 0; q < dev->num_tx_queues; q++) {
2383                ring = &priv->tx_rings[q];
2384
2385                if (ring->switch_port != port)
2386                        continue;
2387
2388                if (!ring->inspect)
2389                        continue;
2390
2391                ring->inspect = false;
2392                qp = ring->switch_queue;
2393                priv->ring_map[qp + port * num_tx_queues] = NULL;
2394        }
2395
2396        return 0;
2397}
2398
2399static int bcm_sysport_netdevice_event(struct notifier_block *nb,
2400                                       unsigned long event, void *ptr)
2401{
2402        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2403        struct netdev_notifier_changeupper_info *info = ptr;
2404        struct bcm_sysport_priv *priv;
2405        int ret = 0;
2406
2407        priv = container_of(nb, struct bcm_sysport_priv, netdev_notifier);
2408        if (priv->netdev != dev)
2409                return NOTIFY_DONE;
2410
2411        switch (event) {
2412        case NETDEV_CHANGEUPPER:
2413                if (dev->netdev_ops != &bcm_sysport_netdev_ops)
2414                        return NOTIFY_DONE;
2415
2416                if (!dsa_slave_dev_check(info->upper_dev))
2417                        return NOTIFY_DONE;
2418
2419                if (info->linking)
2420                        ret = bcm_sysport_map_queues(dev, info->upper_dev);
2421                else
2422                        ret = bcm_sysport_unmap_queues(dev, info->upper_dev);
2423                break;
2424        }
2425
2426        return notifier_from_errno(ret);
2427}
2428
2429#define REV_FMT "v%2x.%02x"
2430
2431static const struct bcm_sysport_hw_params bcm_sysport_params[] = {
2432        [SYSTEMPORT] = {
2433                .is_lite = false,
2434                .num_rx_desc_words = SP_NUM_HW_RX_DESC_WORDS,
2435        },
2436        [SYSTEMPORT_LITE] = {
2437                .is_lite = true,
2438                .num_rx_desc_words = SP_LT_NUM_HW_RX_DESC_WORDS,
2439        },
2440};
2441
2442static const struct of_device_id bcm_sysport_of_match[] = {
2443        { .compatible = "brcm,systemportlite-v1.00",
2444          .data = &bcm_sysport_params[SYSTEMPORT_LITE] },
2445        { .compatible = "brcm,systemport-v1.00",
2446          .data = &bcm_sysport_params[SYSTEMPORT] },
2447        { .compatible = "brcm,systemport",
2448          .data = &bcm_sysport_params[SYSTEMPORT] },
2449        { /* sentinel */ }
2450};
2451MODULE_DEVICE_TABLE(of, bcm_sysport_of_match);
2452
2453static int bcm_sysport_probe(struct platform_device *pdev)
2454{
2455        const struct bcm_sysport_hw_params *params;
2456        const struct of_device_id *of_id = NULL;
2457        struct bcm_sysport_priv *priv;
2458        struct device_node *dn;
2459        struct net_device *dev;
2460        u32 txq, rxq;
2461        int ret;
2462
2463        dn = pdev->dev.of_node;
2464        of_id = of_match_node(bcm_sysport_of_match, dn);
2465        if (!of_id || !of_id->data)
2466                return -EINVAL;
2467
2468        ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
2469        if (ret)
2470                ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2471        if (ret) {
2472                dev_err(&pdev->dev, "unable to set DMA mask: %d\n", ret);
2473                return ret;
2474        }
2475
2476        /* Fairly quickly we need to know the type of adapter we have */
2477        params = of_id->data;
2478
2479        /* Read the Transmit/Receive Queue properties */
2480        if (of_property_read_u32(dn, "systemport,num-txq", &txq))
2481                txq = TDMA_NUM_RINGS;
2482        if (of_property_read_u32(dn, "systemport,num-rxq", &rxq))
2483                rxq = 1;
2484
2485        /* Sanity check the number of transmit queues */
2486        if (!txq || txq > TDMA_NUM_RINGS)
2487                return -EINVAL;
2488
2489        dev = alloc_etherdev_mqs(sizeof(*priv), txq, rxq);
2490        if (!dev)
2491                return -ENOMEM;
2492
2493        /* Initialize private members */
2494        priv = netdev_priv(dev);
2495
2496        priv->clk = devm_clk_get_optional(&pdev->dev, "sw_sysport");
2497        if (IS_ERR(priv->clk)) {
2498                ret = PTR_ERR(priv->clk);
2499                goto err_free_netdev;
2500        }
2501
2502        /* Allocate number of TX rings */
2503        priv->tx_rings = devm_kcalloc(&pdev->dev, txq,
2504                                      sizeof(struct bcm_sysport_tx_ring),
2505                                      GFP_KERNEL);
2506        if (!priv->tx_rings) {
2507                ret = -ENOMEM;
2508                goto err_free_netdev;
2509        }
2510
2511        priv->is_lite = params->is_lite;
2512        priv->num_rx_desc_words = params->num_rx_desc_words;
2513
2514        priv->irq0 = platform_get_irq(pdev, 0);
2515        if (!priv->is_lite) {
2516                priv->irq1 = platform_get_irq(pdev, 1);
2517                priv->wol_irq = platform_get_irq(pdev, 2);
2518        } else {
2519                priv->wol_irq = platform_get_irq(pdev, 1);
2520        }
2521        if (priv->irq0 <= 0 || (priv->irq1 <= 0 && !priv->is_lite)) {
2522                ret = -EINVAL;
2523                goto err_free_netdev;
2524        }
2525
2526        priv->base = devm_platform_ioremap_resource(pdev, 0);
2527        if (IS_ERR(priv->base)) {
2528                ret = PTR_ERR(priv->base);
2529                goto err_free_netdev;
2530        }
2531
2532        priv->netdev = dev;
2533        priv->pdev = pdev;
2534
2535        ret = of_get_phy_mode(dn, &priv->phy_interface);
2536        /* Default to GMII interface mode */
2537        if (ret)
2538                priv->phy_interface = PHY_INTERFACE_MODE_GMII;
2539
2540        /* In the case of a fixed PHY, the DT node associated
2541         * to the PHY is the Ethernet MAC DT node.
2542         */
2543        if (of_phy_is_fixed_link(dn)) {
2544                ret = of_phy_register_fixed_link(dn);
2545                if (ret) {
2546                        dev_err(&pdev->dev, "failed to register fixed PHY\n");
2547                        goto err_free_netdev;
2548                }
2549
2550                priv->phy_dn = dn;
2551        }
2552
2553        /* Initialize netdevice members */
2554        ret = of_get_mac_address(dn, dev->dev_addr);
2555        if (ret) {
2556                dev_warn(&pdev->dev, "using random Ethernet MAC\n");
2557                eth_hw_addr_random(dev);
2558        }
2559
2560        SET_NETDEV_DEV(dev, &pdev->dev);
2561        dev_set_drvdata(&pdev->dev, dev);
2562        dev->ethtool_ops = &bcm_sysport_ethtool_ops;
2563        dev->netdev_ops = &bcm_sysport_netdev_ops;
2564        netif_napi_add(dev, &priv->napi, bcm_sysport_poll, 64);
2565
2566        dev->features |= NETIF_F_RXCSUM | NETIF_F_HIGHDMA |
2567                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2568                         NETIF_F_HW_VLAN_CTAG_TX;
2569        dev->hw_features |= dev->features;
2570        dev->vlan_features |= dev->features;
2571        dev->max_mtu = UMAC_MAX_MTU_SIZE;
2572
2573        /* Request the WOL interrupt and advertise suspend if available */
2574        priv->wol_irq_disabled = 1;
2575        ret = devm_request_irq(&pdev->dev, priv->wol_irq,
2576                               bcm_sysport_wol_isr, 0, dev->name, priv);
2577        if (!ret)
2578                device_set_wakeup_capable(&pdev->dev, 1);
2579
2580        priv->wol_clk = devm_clk_get_optional(&pdev->dev, "sw_sysportwol");
2581        if (IS_ERR(priv->wol_clk))
2582                return PTR_ERR(priv->wol_clk);
2583
2584        /* Set the needed headroom once and for all */
2585        BUILD_BUG_ON(sizeof(struct bcm_tsb) != 8);
2586        dev->needed_headroom += sizeof(struct bcm_tsb);
2587
2588        /* libphy will adjust the link state accordingly */
2589        netif_carrier_off(dev);
2590
2591        priv->rx_max_coalesced_frames = 1;
2592        u64_stats_init(&priv->syncp);
2593
2594        priv->netdev_notifier.notifier_call = bcm_sysport_netdevice_event;
2595
2596        ret = register_netdevice_notifier(&priv->netdev_notifier);
2597        if (ret) {
2598                dev_err(&pdev->dev, "failed to register DSA notifier\n");
2599                goto err_deregister_fixed_link;
2600        }
2601
2602        ret = register_netdev(dev);
2603        if (ret) {
2604                dev_err(&pdev->dev, "failed to register net_device\n");
2605                goto err_deregister_notifier;
2606        }
2607
2608        clk_prepare_enable(priv->clk);
2609
2610        priv->rev = topctrl_readl(priv, REV_CNTL) & REV_MASK;
2611        dev_info(&pdev->dev,
2612                 "Broadcom SYSTEMPORT%s " REV_FMT
2613                 " (irqs: %d, %d, TXQs: %d, RXQs: %d)\n",
2614                 priv->is_lite ? " Lite" : "",
2615                 (priv->rev >> 8) & 0xff, priv->rev & 0xff,
2616                 priv->irq0, priv->irq1, txq, rxq);
2617
2618        clk_disable_unprepare(priv->clk);
2619
2620        return 0;
2621
2622err_deregister_notifier:
2623        unregister_netdevice_notifier(&priv->netdev_notifier);
2624err_deregister_fixed_link:
2625        if (of_phy_is_fixed_link(dn))
2626                of_phy_deregister_fixed_link(dn);
2627err_free_netdev:
2628        free_netdev(dev);
2629        return ret;
2630}
2631
2632static int bcm_sysport_remove(struct platform_device *pdev)
2633{
2634        struct net_device *dev = dev_get_drvdata(&pdev->dev);
2635        struct bcm_sysport_priv *priv = netdev_priv(dev);
2636        struct device_node *dn = pdev->dev.of_node;
2637
2638        /* Not much to do, ndo_close has been called
2639         * and we use managed allocations
2640         */
2641        unregister_netdevice_notifier(&priv->netdev_notifier);
2642        unregister_netdev(dev);
2643        if (of_phy_is_fixed_link(dn))
2644                of_phy_deregister_fixed_link(dn);
2645        free_netdev(dev);
2646        dev_set_drvdata(&pdev->dev, NULL);
2647
2648        return 0;
2649}
2650
2651static int bcm_sysport_suspend_to_wol(struct bcm_sysport_priv *priv)
2652{
2653        struct net_device *ndev = priv->netdev;
2654        unsigned int timeout = 1000;
2655        unsigned int index, i = 0;
2656        u32 reg;
2657
2658        reg = umac_readl(priv, UMAC_MPD_CTRL);
2659        if (priv->wolopts & (WAKE_MAGIC | WAKE_MAGICSECURE))
2660                reg |= MPD_EN;
2661        reg &= ~PSW_EN;
2662        if (priv->wolopts & WAKE_MAGICSECURE) {
2663                /* Program the SecureOn password */
2664                umac_writel(priv, get_unaligned_be16(&priv->sopass[0]),
2665                            UMAC_PSW_MS);
2666                umac_writel(priv, get_unaligned_be32(&priv->sopass[2]),
2667                            UMAC_PSW_LS);
2668                reg |= PSW_EN;
2669        }
2670        umac_writel(priv, reg, UMAC_MPD_CTRL);
2671
2672        if (priv->wolopts & WAKE_FILTER) {
2673                /* Turn on ACPI matching to steal packets from RBUF */
2674                reg = rbuf_readl(priv, RBUF_CONTROL);
2675                if (priv->is_lite)
2676                        reg |= RBUF_ACPI_EN_LITE;
2677                else
2678                        reg |= RBUF_ACPI_EN;
2679                rbuf_writel(priv, reg, RBUF_CONTROL);
2680
2681                /* Enable RXCHK, active filters and Broadcom tag matching */
2682                reg = rxchk_readl(priv, RXCHK_CONTROL);
2683                reg &= ~(RXCHK_BRCM_TAG_MATCH_MASK <<
2684                         RXCHK_BRCM_TAG_MATCH_SHIFT);
2685                for_each_set_bit(index, priv->filters, RXCHK_BRCM_TAG_MAX) {
2686                        reg |= BIT(RXCHK_BRCM_TAG_MATCH_SHIFT + i);
2687                        i++;
2688                }
2689                reg |= RXCHK_EN | RXCHK_BRCM_TAG_EN;
2690                rxchk_writel(priv, reg, RXCHK_CONTROL);
2691        }
2692
2693        /* Make sure RBUF entered WoL mode as result */
2694        do {
2695                reg = rbuf_readl(priv, RBUF_STATUS);
2696                if (reg & RBUF_WOL_MODE)
2697                        break;
2698
2699                udelay(10);
2700        } while (timeout-- > 0);
2701
2702        /* Do not leave the UniMAC RBUF matching only MPD packets */
2703        if (!timeout) {
2704                mpd_enable_set(priv, false);
2705                netif_err(priv, wol, ndev, "failed to enter WOL mode\n");
2706                return -ETIMEDOUT;
2707        }
2708
2709        /* UniMAC receive needs to be turned on */
2710        umac_enable_set(priv, CMD_RX_EN, 1);
2711
2712        netif_dbg(priv, wol, ndev, "entered WOL mode\n");
2713
2714        return 0;
2715}
2716
2717static int __maybe_unused bcm_sysport_suspend(struct device *d)
2718{
2719        struct net_device *dev = dev_get_drvdata(d);
2720        struct bcm_sysport_priv *priv = netdev_priv(dev);
2721        unsigned int i;
2722        int ret = 0;
2723        u32 reg;
2724
2725        if (!netif_running(dev))
2726                return 0;
2727
2728        netif_device_detach(dev);
2729
2730        bcm_sysport_netif_stop(dev);
2731
2732        phy_suspend(dev->phydev);
2733
2734        /* Disable UniMAC RX */
2735        umac_enable_set(priv, CMD_RX_EN, 0);
2736
2737        ret = rdma_enable_set(priv, 0);
2738        if (ret) {
2739                netdev_err(dev, "RDMA timeout!\n");
2740                return ret;
2741        }
2742
2743        /* Disable RXCHK if enabled */
2744        if (priv->rx_chk_en) {
2745                reg = rxchk_readl(priv, RXCHK_CONTROL);
2746                reg &= ~RXCHK_EN;
2747                rxchk_writel(priv, reg, RXCHK_CONTROL);
2748        }
2749
2750        /* Flush RX pipe */
2751        if (!priv->wolopts)
2752                topctrl_writel(priv, RX_FLUSH, RX_FLUSH_CNTL);
2753
2754        ret = tdma_enable_set(priv, 0);
2755        if (ret) {
2756                netdev_err(dev, "TDMA timeout!\n");
2757                return ret;
2758        }
2759
2760        /* Wait for a packet boundary */
2761        usleep_range(2000, 3000);
2762
2763        umac_enable_set(priv, CMD_TX_EN, 0);
2764
2765        topctrl_writel(priv, TX_FLUSH, TX_FLUSH_CNTL);
2766
2767        /* Free RX/TX rings SW structures */
2768        for (i = 0; i < dev->num_tx_queues; i++)
2769                bcm_sysport_fini_tx_ring(priv, i);
2770        bcm_sysport_fini_rx_ring(priv);
2771
2772        /* Get prepared for Wake-on-LAN */
2773        if (device_may_wakeup(d) && priv->wolopts) {
2774                clk_prepare_enable(priv->wol_clk);
2775                ret = bcm_sysport_suspend_to_wol(priv);
2776        }
2777
2778        clk_disable_unprepare(priv->clk);
2779
2780        return ret;
2781}
2782
2783static int __maybe_unused bcm_sysport_resume(struct device *d)
2784{
2785        struct net_device *dev = dev_get_drvdata(d);
2786        struct bcm_sysport_priv *priv = netdev_priv(dev);
2787        unsigned int i;
2788        int ret;
2789
2790        if (!netif_running(dev))
2791                return 0;
2792
2793        clk_prepare_enable(priv->clk);
2794        if (priv->wolopts)
2795                clk_disable_unprepare(priv->wol_clk);
2796
2797        umac_reset(priv);
2798
2799        /* Disable the UniMAC RX/TX */
2800        umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, 0);
2801
2802        /* We may have been suspended and never received a WOL event that
2803         * would turn off MPD detection, take care of that now
2804         */
2805        bcm_sysport_resume_from_wol(priv);
2806
2807        /* Initialize both hardware and software ring */
2808        for (i = 0; i < dev->num_tx_queues; i++) {
2809                ret = bcm_sysport_init_tx_ring(priv, i);
2810                if (ret) {
2811                        netdev_err(dev, "failed to initialize TX ring %d\n",
2812                                   i);
2813                        goto out_free_tx_rings;
2814                }
2815        }
2816
2817        /* Initialize linked-list */
2818        tdma_writel(priv, TDMA_LL_RAM_INIT_BUSY, TDMA_STATUS);
2819
2820        /* Initialize RX ring */
2821        ret = bcm_sysport_init_rx_ring(priv);
2822        if (ret) {
2823                netdev_err(dev, "failed to initialize RX ring\n");
2824                goto out_free_rx_ring;
2825        }
2826
2827        /* RX pipe enable */
2828        topctrl_writel(priv, 0, RX_FLUSH_CNTL);
2829
2830        ret = rdma_enable_set(priv, 1);
2831        if (ret) {
2832                netdev_err(dev, "failed to enable RDMA\n");
2833                goto out_free_rx_ring;
2834        }
2835
2836        /* Restore enabled features */
2837        bcm_sysport_set_features(dev, dev->features);
2838
2839        rbuf_init(priv);
2840
2841        /* Set maximum frame length */
2842        if (!priv->is_lite)
2843                umac_writel(priv, UMAC_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
2844        else
2845                gib_set_pad_extension(priv);
2846
2847        /* Set MAC address */
2848        umac_set_hw_addr(priv, dev->dev_addr);
2849
2850        umac_enable_set(priv, CMD_RX_EN, 1);
2851
2852        /* TX pipe enable */
2853        topctrl_writel(priv, 0, TX_FLUSH_CNTL);
2854
2855        umac_enable_set(priv, CMD_TX_EN, 1);
2856
2857        ret = tdma_enable_set(priv, 1);
2858        if (ret) {
2859                netdev_err(dev, "TDMA timeout!\n");
2860                goto out_free_rx_ring;
2861        }
2862
2863        phy_resume(dev->phydev);
2864
2865        bcm_sysport_netif_start(dev);
2866
2867        netif_device_attach(dev);
2868
2869        return 0;
2870
2871out_free_rx_ring:
2872        bcm_sysport_fini_rx_ring(priv);
2873out_free_tx_rings:
2874        for (i = 0; i < dev->num_tx_queues; i++)
2875                bcm_sysport_fini_tx_ring(priv, i);
2876        clk_disable_unprepare(priv->clk);
2877        return ret;
2878}
2879
2880static SIMPLE_DEV_PM_OPS(bcm_sysport_pm_ops,
2881                bcm_sysport_suspend, bcm_sysport_resume);
2882
2883static struct platform_driver bcm_sysport_driver = {
2884        .probe  = bcm_sysport_probe,
2885        .remove = bcm_sysport_remove,
2886        .driver =  {
2887                .name = "brcm-systemport",
2888                .of_match_table = bcm_sysport_of_match,
2889                .pm = &bcm_sysport_pm_ops,
2890        },
2891};
2892module_platform_driver(bcm_sysport_driver);
2893
2894MODULE_AUTHOR("Broadcom Corporation");
2895MODULE_DESCRIPTION("Broadcom System Port Ethernet MAC driver");
2896MODULE_ALIAS("platform:brcm-systemport");
2897MODULE_LICENSE("GPL");
2898