linux/drivers/pci/controller/pcie-rockchip-host.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Rockchip AXI PCIe host controller driver
   4 *
   5 * Copyright (c) 2016 Rockchip, Inc.
   6 *
   7 * Author: Shawn Lin <shawn.lin@rock-chips.com>
   8 *         Wenrui Li <wenrui.li@rock-chips.com>
   9 *
  10 * Bits taken from Synopsys DesignWare Host controller driver and
  11 * ARM PCI Host generic driver.
  12 */
  13
  14#include <linux/bitrev.h>
  15#include <linux/clk.h>
  16#include <linux/delay.h>
  17#include <linux/gpio/consumer.h>
  18#include <linux/init.h>
  19#include <linux/interrupt.h>
  20#include <linux/iopoll.h>
  21#include <linux/irq.h>
  22#include <linux/irqchip/chained_irq.h>
  23#include <linux/irqdomain.h>
  24#include <linux/kernel.h>
  25#include <linux/mfd/syscon.h>
  26#include <linux/module.h>
  27#include <linux/of_address.h>
  28#include <linux/of_device.h>
  29#include <linux/of_pci.h>
  30#include <linux/of_platform.h>
  31#include <linux/of_irq.h>
  32#include <linux/pci.h>
  33#include <linux/pci_ids.h>
  34#include <linux/phy/phy.h>
  35#include <linux/platform_device.h>
  36#include <linux/reset.h>
  37#include <linux/regmap.h>
  38
  39#include "../pci.h"
  40#include "pcie-rockchip.h"
  41
  42static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
  43{
  44        u32 status;
  45
  46        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  47        status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
  48        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  49}
  50
  51static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
  52{
  53        u32 status;
  54
  55        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  56        status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
  57        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  58}
  59
  60static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
  61{
  62        u32 val;
  63
  64        /* Update Tx credit maximum update interval */
  65        val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
  66        val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
  67        val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000);       /* ns */
  68        rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
  69}
  70
  71static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
  72                                      struct pci_bus *bus, int dev)
  73{
  74        /*
  75         * Access only one slot on each root port.
  76         * Do not read more than one device on the bus directly attached
  77         * to RC's downstream side.
  78         */
  79        if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent))
  80                return dev == 0;
  81
  82        return 1;
  83}
  84
  85static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
  86{
  87        u32 val;
  88        u8 map;
  89
  90        if (rockchip->legacy_phy)
  91                return GENMASK(MAX_LANE_NUM - 1, 0);
  92
  93        val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
  94        map = val & PCIE_CORE_LANE_MAP_MASK;
  95
  96        /* The link may be using a reverse-indexed mapping. */
  97        if (val & PCIE_CORE_LANE_MAP_REVERSE)
  98                map = bitrev8(map) >> 4;
  99
 100        return map;
 101}
 102
 103static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
 104                                     int where, int size, u32 *val)
 105{
 106        void __iomem *addr;
 107
 108        addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
 109
 110        if (!IS_ALIGNED((uintptr_t)addr, size)) {
 111                *val = 0;
 112                return PCIBIOS_BAD_REGISTER_NUMBER;
 113        }
 114
 115        if (size == 4) {
 116                *val = readl(addr);
 117        } else if (size == 2) {
 118                *val = readw(addr);
 119        } else if (size == 1) {
 120                *val = readb(addr);
 121        } else {
 122                *val = 0;
 123                return PCIBIOS_BAD_REGISTER_NUMBER;
 124        }
 125        return PCIBIOS_SUCCESSFUL;
 126}
 127
 128static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
 129                                     int where, int size, u32 val)
 130{
 131        u32 mask, tmp, offset;
 132        void __iomem *addr;
 133
 134        offset = where & ~0x3;
 135        addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
 136
 137        if (size == 4) {
 138                writel(val, addr);
 139                return PCIBIOS_SUCCESSFUL;
 140        }
 141
 142        mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 143
 144        /*
 145         * N.B. This read/modify/write isn't safe in general because it can
 146         * corrupt RW1C bits in adjacent registers.  But the hardware
 147         * doesn't support smaller writes.
 148         */
 149        tmp = readl(addr) & mask;
 150        tmp |= val << ((where & 0x3) * 8);
 151        writel(tmp, addr);
 152
 153        return PCIBIOS_SUCCESSFUL;
 154}
 155
 156static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
 157                                       struct pci_bus *bus, u32 devfn,
 158                                       int where, int size, u32 *val)
 159{
 160        void __iomem *addr;
 161
 162        addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);
 163
 164        if (!IS_ALIGNED((uintptr_t)addr, size)) {
 165                *val = 0;
 166                return PCIBIOS_BAD_REGISTER_NUMBER;
 167        }
 168
 169        if (pci_is_root_bus(bus->parent))
 170                rockchip_pcie_cfg_configuration_accesses(rockchip,
 171                                                AXI_WRAPPER_TYPE0_CFG);
 172        else
 173                rockchip_pcie_cfg_configuration_accesses(rockchip,
 174                                                AXI_WRAPPER_TYPE1_CFG);
 175
 176        if (size == 4) {
 177                *val = readl(addr);
 178        } else if (size == 2) {
 179                *val = readw(addr);
 180        } else if (size == 1) {
 181                *val = readb(addr);
 182        } else {
 183                *val = 0;
 184                return PCIBIOS_BAD_REGISTER_NUMBER;
 185        }
 186        return PCIBIOS_SUCCESSFUL;
 187}
 188
 189static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
 190                                       struct pci_bus *bus, u32 devfn,
 191                                       int where, int size, u32 val)
 192{
 193        void __iomem *addr;
 194
 195        addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);
 196
 197        if (!IS_ALIGNED((uintptr_t)addr, size))
 198                return PCIBIOS_BAD_REGISTER_NUMBER;
 199
 200        if (pci_is_root_bus(bus->parent))
 201                rockchip_pcie_cfg_configuration_accesses(rockchip,
 202                                                AXI_WRAPPER_TYPE0_CFG);
 203        else
 204                rockchip_pcie_cfg_configuration_accesses(rockchip,
 205                                                AXI_WRAPPER_TYPE1_CFG);
 206
 207        if (size == 4)
 208                writel(val, addr);
 209        else if (size == 2)
 210                writew(val, addr);
 211        else if (size == 1)
 212                writeb(val, addr);
 213        else
 214                return PCIBIOS_BAD_REGISTER_NUMBER;
 215
 216        return PCIBIOS_SUCCESSFUL;
 217}
 218
 219static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
 220                                 int size, u32 *val)
 221{
 222        struct rockchip_pcie *rockchip = bus->sysdata;
 223
 224        if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
 225                *val = 0xffffffff;
 226                return PCIBIOS_DEVICE_NOT_FOUND;
 227        }
 228
 229        if (pci_is_root_bus(bus))
 230                return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
 231
 232        return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size,
 233                                           val);
 234}
 235
 236static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
 237                                 int where, int size, u32 val)
 238{
 239        struct rockchip_pcie *rockchip = bus->sysdata;
 240
 241        if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
 242                return PCIBIOS_DEVICE_NOT_FOUND;
 243
 244        if (pci_is_root_bus(bus))
 245                return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
 246
 247        return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size,
 248                                           val);
 249}
 250
 251static struct pci_ops rockchip_pcie_ops = {
 252        .read = rockchip_pcie_rd_conf,
 253        .write = rockchip_pcie_wr_conf,
 254};
 255
 256static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
 257{
 258        int curr;
 259        u32 status, scale, power;
 260
 261        if (IS_ERR(rockchip->vpcie3v3))
 262                return;
 263
 264        /*
 265         * Set RC's captured slot power limit and scale if
 266         * vpcie3v3 available. The default values are both zero
 267         * which means the software should set these two according
 268         * to the actual power supply.
 269         */
 270        curr = regulator_get_current_limit(rockchip->vpcie3v3);
 271        if (curr <= 0)
 272                return;
 273
 274        scale = 3; /* 0.001x */
 275        curr = curr / 1000; /* convert to mA */
 276        power = (curr * 3300) / 1000; /* milliwatt */
 277        while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
 278                if (!scale) {
 279                        dev_warn(rockchip->dev, "invalid power supply\n");
 280                        return;
 281                }
 282                scale--;
 283                power = power / 10;
 284        }
 285
 286        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
 287        status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
 288                  (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
 289        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
 290}
 291
 292/**
 293 * rockchip_pcie_host_init_port - Initialize hardware
 294 * @rockchip: PCIe port information
 295 */
 296static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip)
 297{
 298        struct device *dev = rockchip->dev;
 299        int err, i = MAX_LANE_NUM;
 300        u32 status;
 301
 302        gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
 303
 304        err = rockchip_pcie_init_port(rockchip);
 305        if (err)
 306                return err;
 307
 308        /* Fix the transmitted FTS count desired to exit from L0s. */
 309        status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
 310        status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
 311                 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
 312        rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
 313
 314        rockchip_pcie_set_power_limit(rockchip);
 315
 316        /* Set RC's clock architecture as common clock */
 317        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
 318        status |= PCI_EXP_LNKSTA_SLC << 16;
 319        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
 320
 321        /* Set RC's RCB to 128 */
 322        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
 323        status |= PCI_EXP_LNKCTL_RCB;
 324        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
 325
 326        /* Enable Gen1 training */
 327        rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
 328                            PCIE_CLIENT_CONFIG);
 329
 330        gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
 331
 332        /* 500ms timeout value should be enough for Gen1/2 training */
 333        err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
 334                                 status, PCIE_LINK_UP(status), 20,
 335                                 500 * USEC_PER_MSEC);
 336        if (err) {
 337                dev_err(dev, "PCIe link training gen1 timeout!\n");
 338                goto err_power_off_phy;
 339        }
 340
 341        if (rockchip->link_gen == 2) {
 342                /*
 343                 * Enable retrain for gen2. This should be configured only after
 344                 * gen1 finished.
 345                 */
 346                status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
 347                status |= PCI_EXP_LNKCTL_RL;
 348                rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
 349
 350                err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
 351                                         status, PCIE_LINK_IS_GEN2(status), 20,
 352                                         500 * USEC_PER_MSEC);
 353                if (err)
 354                        dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
 355        }
 356
 357        /* Check the final link width from negotiated lane counter from MGMT */
 358        status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
 359        status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
 360                          PCIE_CORE_PL_CONF_LANE_SHIFT);
 361        dev_dbg(dev, "current link width is x%d\n", status);
 362
 363        /* Power off unused lane(s) */
 364        rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
 365        for (i = 0; i < MAX_LANE_NUM; i++) {
 366                if (!(rockchip->lanes_map & BIT(i))) {
 367                        dev_dbg(dev, "idling lane %d\n", i);
 368                        phy_power_off(rockchip->phys[i]);
 369                }
 370        }
 371
 372        rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
 373                            PCIE_CORE_CONFIG_VENDOR);
 374        rockchip_pcie_write(rockchip,
 375                            PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
 376                            PCIE_RC_CONFIG_RID_CCR);
 377
 378        /* Clear THP cap's next cap pointer to remove L1 substate cap */
 379        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
 380        status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
 381        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
 382
 383        /* Clear L0s from RC's link cap */
 384        if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
 385                status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
 386                status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
 387                rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
 388        }
 389
 390        status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
 391        status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
 392        status |= PCIE_RC_CONFIG_DCSR_MPS_256;
 393        rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
 394
 395        return 0;
 396err_power_off_phy:
 397        while (i--)
 398                phy_power_off(rockchip->phys[i]);
 399        i = MAX_LANE_NUM;
 400        while (i--)
 401                phy_exit(rockchip->phys[i]);
 402        return err;
 403}
 404
 405static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
 406{
 407        struct rockchip_pcie *rockchip = arg;
 408        struct device *dev = rockchip->dev;
 409        u32 reg;
 410        u32 sub_reg;
 411
 412        reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
 413        if (reg & PCIE_CLIENT_INT_LOCAL) {
 414                dev_dbg(dev, "local interrupt received\n");
 415                sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
 416                if (sub_reg & PCIE_CORE_INT_PRFPE)
 417                        dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
 418
 419                if (sub_reg & PCIE_CORE_INT_CRFPE)
 420                        dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
 421
 422                if (sub_reg & PCIE_CORE_INT_RRPE)
 423                        dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
 424
 425                if (sub_reg & PCIE_CORE_INT_PRFO)
 426                        dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
 427
 428                if (sub_reg & PCIE_CORE_INT_CRFO)
 429                        dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
 430
 431                if (sub_reg & PCIE_CORE_INT_RT)
 432                        dev_dbg(dev, "replay timer timed out\n");
 433
 434                if (sub_reg & PCIE_CORE_INT_RTR)
 435                        dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
 436
 437                if (sub_reg & PCIE_CORE_INT_PE)
 438                        dev_dbg(dev, "phy error detected on receive side\n");
 439
 440                if (sub_reg & PCIE_CORE_INT_MTR)
 441                        dev_dbg(dev, "malformed TLP received from the link\n");
 442
 443                if (sub_reg & PCIE_CORE_INT_UCR)
 444                        dev_dbg(dev, "malformed TLP received from the link\n");
 445
 446                if (sub_reg & PCIE_CORE_INT_FCE)
 447                        dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
 448
 449                if (sub_reg & PCIE_CORE_INT_CT)
 450                        dev_dbg(dev, "a request timed out waiting for completion\n");
 451
 452                if (sub_reg & PCIE_CORE_INT_UTC)
 453                        dev_dbg(dev, "unmapped TC error\n");
 454
 455                if (sub_reg & PCIE_CORE_INT_MMVC)
 456                        dev_dbg(dev, "MSI mask register changes\n");
 457
 458                rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
 459        } else if (reg & PCIE_CLIENT_INT_PHY) {
 460                dev_dbg(dev, "phy link changes\n");
 461                rockchip_pcie_update_txcredit_mui(rockchip);
 462                rockchip_pcie_clr_bw_int(rockchip);
 463        }
 464
 465        rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
 466                            PCIE_CLIENT_INT_STATUS);
 467
 468        return IRQ_HANDLED;
 469}
 470
 471static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
 472{
 473        struct rockchip_pcie *rockchip = arg;
 474        struct device *dev = rockchip->dev;
 475        u32 reg;
 476
 477        reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
 478        if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
 479                dev_dbg(dev, "legacy done interrupt received\n");
 480
 481        if (reg & PCIE_CLIENT_INT_MSG)
 482                dev_dbg(dev, "message done interrupt received\n");
 483
 484        if (reg & PCIE_CLIENT_INT_HOT_RST)
 485                dev_dbg(dev, "hot reset interrupt received\n");
 486
 487        if (reg & PCIE_CLIENT_INT_DPA)
 488                dev_dbg(dev, "dpa interrupt received\n");
 489
 490        if (reg & PCIE_CLIENT_INT_FATAL_ERR)
 491                dev_dbg(dev, "fatal error interrupt received\n");
 492
 493        if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
 494                dev_dbg(dev, "no fatal error interrupt received\n");
 495
 496        if (reg & PCIE_CLIENT_INT_CORR_ERR)
 497                dev_dbg(dev, "correctable error interrupt received\n");
 498
 499        if (reg & PCIE_CLIENT_INT_PHY)
 500                dev_dbg(dev, "phy interrupt received\n");
 501
 502        rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
 503                              PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
 504                              PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
 505                              PCIE_CLIENT_INT_NFATAL_ERR |
 506                              PCIE_CLIENT_INT_CORR_ERR |
 507                              PCIE_CLIENT_INT_PHY),
 508                   PCIE_CLIENT_INT_STATUS);
 509
 510        return IRQ_HANDLED;
 511}
 512
 513static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
 514{
 515        struct irq_chip *chip = irq_desc_get_chip(desc);
 516        struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
 517        struct device *dev = rockchip->dev;
 518        u32 reg;
 519        u32 hwirq;
 520        u32 virq;
 521
 522        chained_irq_enter(chip, desc);
 523
 524        reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
 525        reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
 526
 527        while (reg) {
 528                hwirq = ffs(reg) - 1;
 529                reg &= ~BIT(hwirq);
 530
 531                virq = irq_find_mapping(rockchip->irq_domain, hwirq);
 532                if (virq)
 533                        generic_handle_irq(virq);
 534                else
 535                        dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
 536        }
 537
 538        chained_irq_exit(chip, desc);
 539}
 540
 541static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
 542{
 543        int irq, err;
 544        struct device *dev = rockchip->dev;
 545        struct platform_device *pdev = to_platform_device(dev);
 546
 547        irq = platform_get_irq_byname(pdev, "sys");
 548        if (irq < 0)
 549                return irq;
 550
 551        err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
 552                               IRQF_SHARED, "pcie-sys", rockchip);
 553        if (err) {
 554                dev_err(dev, "failed to request PCIe subsystem IRQ\n");
 555                return err;
 556        }
 557
 558        irq = platform_get_irq_byname(pdev, "legacy");
 559        if (irq < 0)
 560                return irq;
 561
 562        irq_set_chained_handler_and_data(irq,
 563                                         rockchip_pcie_legacy_int_handler,
 564                                         rockchip);
 565
 566        irq = platform_get_irq_byname(pdev, "client");
 567        if (irq < 0)
 568                return irq;
 569
 570        err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
 571                               IRQF_SHARED, "pcie-client", rockchip);
 572        if (err) {
 573                dev_err(dev, "failed to request PCIe client IRQ\n");
 574                return err;
 575        }
 576
 577        return 0;
 578}
 579
 580/**
 581 * rockchip_pcie_parse_host_dt - Parse Device Tree
 582 * @rockchip: PCIe port information
 583 *
 584 * Return: '0' on success and error value on failure
 585 */
 586static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip)
 587{
 588        struct device *dev = rockchip->dev;
 589        int err;
 590
 591        err = rockchip_pcie_parse_dt(rockchip);
 592        if (err)
 593                return err;
 594
 595        rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
 596        if (IS_ERR(rockchip->vpcie12v)) {
 597                if (PTR_ERR(rockchip->vpcie12v) != -ENODEV)
 598                        return PTR_ERR(rockchip->vpcie12v);
 599                dev_info(dev, "no vpcie12v regulator found\n");
 600        }
 601
 602        rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
 603        if (IS_ERR(rockchip->vpcie3v3)) {
 604                if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)
 605                        return PTR_ERR(rockchip->vpcie3v3);
 606                dev_info(dev, "no vpcie3v3 regulator found\n");
 607        }
 608
 609        rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8");
 610        if (IS_ERR(rockchip->vpcie1v8))
 611                return PTR_ERR(rockchip->vpcie1v8);
 612
 613        rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9");
 614        if (IS_ERR(rockchip->vpcie0v9))
 615                return PTR_ERR(rockchip->vpcie0v9);
 616
 617        return 0;
 618}
 619
 620static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
 621{
 622        struct device *dev = rockchip->dev;
 623        int err;
 624
 625        if (!IS_ERR(rockchip->vpcie12v)) {
 626                err = regulator_enable(rockchip->vpcie12v);
 627                if (err) {
 628                        dev_err(dev, "fail to enable vpcie12v regulator\n");
 629                        goto err_out;
 630                }
 631        }
 632
 633        if (!IS_ERR(rockchip->vpcie3v3)) {
 634                err = regulator_enable(rockchip->vpcie3v3);
 635                if (err) {
 636                        dev_err(dev, "fail to enable vpcie3v3 regulator\n");
 637                        goto err_disable_12v;
 638                }
 639        }
 640
 641        err = regulator_enable(rockchip->vpcie1v8);
 642        if (err) {
 643                dev_err(dev, "fail to enable vpcie1v8 regulator\n");
 644                goto err_disable_3v3;
 645        }
 646
 647        err = regulator_enable(rockchip->vpcie0v9);
 648        if (err) {
 649                dev_err(dev, "fail to enable vpcie0v9 regulator\n");
 650                goto err_disable_1v8;
 651        }
 652
 653        return 0;
 654
 655err_disable_1v8:
 656        regulator_disable(rockchip->vpcie1v8);
 657err_disable_3v3:
 658        if (!IS_ERR(rockchip->vpcie3v3))
 659                regulator_disable(rockchip->vpcie3v3);
 660err_disable_12v:
 661        if (!IS_ERR(rockchip->vpcie12v))
 662                regulator_disable(rockchip->vpcie12v);
 663err_out:
 664        return err;
 665}
 666
 667static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
 668{
 669        rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
 670                            (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
 671        rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
 672                            PCIE_CORE_INT_MASK);
 673
 674        rockchip_pcie_enable_bw_int(rockchip);
 675}
 676
 677static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 678                                  irq_hw_number_t hwirq)
 679{
 680        irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
 681        irq_set_chip_data(irq, domain->host_data);
 682
 683        return 0;
 684}
 685
 686static const struct irq_domain_ops intx_domain_ops = {
 687        .map = rockchip_pcie_intx_map,
 688};
 689
 690static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
 691{
 692        struct device *dev = rockchip->dev;
 693        struct device_node *intc = of_get_next_child(dev->of_node, NULL);
 694
 695        if (!intc) {
 696                dev_err(dev, "missing child interrupt-controller node\n");
 697                return -EINVAL;
 698        }
 699
 700        rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
 701                                                    &intx_domain_ops, rockchip);
 702        of_node_put(intc);
 703        if (!rockchip->irq_domain) {
 704                dev_err(dev, "failed to get a INTx IRQ domain\n");
 705                return -EINVAL;
 706        }
 707
 708        return 0;
 709}
 710
 711static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
 712                                     int region_no, int type, u8 num_pass_bits,
 713                                     u32 lower_addr, u32 upper_addr)
 714{
 715        u32 ob_addr_0;
 716        u32 ob_addr_1;
 717        u32 ob_desc_0;
 718        u32 aw_offset;
 719
 720        if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
 721                return -EINVAL;
 722        if (num_pass_bits + 1 < 8)
 723                return -EINVAL;
 724        if (num_pass_bits > 63)
 725                return -EINVAL;
 726        if (region_no == 0) {
 727                if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
 728                        return -EINVAL;
 729        }
 730        if (region_no != 0) {
 731                if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
 732                        return -EINVAL;
 733        }
 734
 735        aw_offset = (region_no << OB_REG_SIZE_SHIFT);
 736
 737        ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
 738        ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
 739        ob_addr_1 = upper_addr;
 740        ob_desc_0 = (1 << 23 | type);
 741
 742        rockchip_pcie_write(rockchip, ob_addr_0,
 743                            PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
 744        rockchip_pcie_write(rockchip, ob_addr_1,
 745                            PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
 746        rockchip_pcie_write(rockchip, ob_desc_0,
 747                            PCIE_CORE_OB_REGION_DESC0 + aw_offset);
 748        rockchip_pcie_write(rockchip, 0,
 749                            PCIE_CORE_OB_REGION_DESC1 + aw_offset);
 750
 751        return 0;
 752}
 753
 754static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
 755                                     int region_no, u8 num_pass_bits,
 756                                     u32 lower_addr, u32 upper_addr)
 757{
 758        u32 ib_addr_0;
 759        u32 ib_addr_1;
 760        u32 aw_offset;
 761
 762        if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
 763                return -EINVAL;
 764        if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
 765                return -EINVAL;
 766        if (num_pass_bits > 63)
 767                return -EINVAL;
 768
 769        aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
 770
 771        ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
 772        ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
 773        ib_addr_1 = upper_addr;
 774
 775        rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
 776        rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
 777
 778        return 0;
 779}
 780
 781static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
 782{
 783        struct device *dev = rockchip->dev;
 784        struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
 785        struct resource_entry *entry;
 786        u64 pci_addr, size;
 787        int offset;
 788        int err;
 789        int reg_no;
 790
 791        rockchip_pcie_cfg_configuration_accesses(rockchip,
 792                                                 AXI_WRAPPER_TYPE0_CFG);
 793        entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM);
 794        if (!entry)
 795                return -ENODEV;
 796
 797        size = resource_size(entry->res);
 798        pci_addr = entry->res->start - entry->offset;
 799        rockchip->msg_bus_addr = pci_addr;
 800
 801        for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
 802                err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
 803                                                AXI_WRAPPER_MEM_WRITE,
 804                                                20 - 1,
 805                                                pci_addr + (reg_no << 20),
 806                                                0);
 807                if (err) {
 808                        dev_err(dev, "program RC mem outbound ATU failed\n");
 809                        return err;
 810                }
 811        }
 812
 813        err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
 814        if (err) {
 815                dev_err(dev, "program RC mem inbound ATU failed\n");
 816                return err;
 817        }
 818
 819        entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
 820        if (!entry)
 821                return -ENODEV;
 822
 823        /* store the register number offset to program RC io outbound ATU */
 824        offset = size >> 20;
 825
 826        size = resource_size(entry->res);
 827        pci_addr = entry->res->start - entry->offset;
 828
 829        for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
 830                err = rockchip_pcie_prog_ob_atu(rockchip,
 831                                                reg_no + 1 + offset,
 832                                                AXI_WRAPPER_IO_WRITE,
 833                                                20 - 1,
 834                                                pci_addr + (reg_no << 20),
 835                                                0);
 836                if (err) {
 837                        dev_err(dev, "program RC io outbound ATU failed\n");
 838                        return err;
 839                }
 840        }
 841
 842        /* assign message regions */
 843        rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
 844                                  AXI_WRAPPER_NOR_MSG,
 845                                  20 - 1, 0, 0);
 846
 847        rockchip->msg_bus_addr += ((reg_no + offset) << 20);
 848        return err;
 849}
 850
 851static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
 852{
 853        u32 value;
 854        int err;
 855
 856        /* send PME_TURN_OFF message */
 857        writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
 858
 859        /* read LTSSM and wait for falling into L2 link state */
 860        err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
 861                                 value, PCIE_LINK_IS_L2(value), 20,
 862                                 jiffies_to_usecs(5 * HZ));
 863        if (err) {
 864                dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
 865                return err;
 866        }
 867
 868        return 0;
 869}
 870
 871static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
 872{
 873        struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
 874        int ret;
 875
 876        /* disable core and cli int since we don't need to ack PME_ACK */
 877        rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
 878                            PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
 879        rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
 880
 881        ret = rockchip_pcie_wait_l2(rockchip);
 882        if (ret) {
 883                rockchip_pcie_enable_interrupts(rockchip);
 884                return ret;
 885        }
 886
 887        rockchip_pcie_deinit_phys(rockchip);
 888
 889        rockchip_pcie_disable_clocks(rockchip);
 890
 891        regulator_disable(rockchip->vpcie0v9);
 892
 893        return ret;
 894}
 895
 896static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
 897{
 898        struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
 899        int err;
 900
 901        err = regulator_enable(rockchip->vpcie0v9);
 902        if (err) {
 903                dev_err(dev, "fail to enable vpcie0v9 regulator\n");
 904                return err;
 905        }
 906
 907        err = rockchip_pcie_enable_clocks(rockchip);
 908        if (err)
 909                goto err_disable_0v9;
 910
 911        err = rockchip_pcie_host_init_port(rockchip);
 912        if (err)
 913                goto err_pcie_resume;
 914
 915        err = rockchip_pcie_cfg_atu(rockchip);
 916        if (err)
 917                goto err_err_deinit_port;
 918
 919        /* Need this to enter L1 again */
 920        rockchip_pcie_update_txcredit_mui(rockchip);
 921        rockchip_pcie_enable_interrupts(rockchip);
 922
 923        return 0;
 924
 925err_err_deinit_port:
 926        rockchip_pcie_deinit_phys(rockchip);
 927err_pcie_resume:
 928        rockchip_pcie_disable_clocks(rockchip);
 929err_disable_0v9:
 930        regulator_disable(rockchip->vpcie0v9);
 931        return err;
 932}
 933
 934static int rockchip_pcie_probe(struct platform_device *pdev)
 935{
 936        struct rockchip_pcie *rockchip;
 937        struct device *dev = &pdev->dev;
 938        struct pci_host_bridge *bridge;
 939        int err;
 940
 941        if (!dev->of_node)
 942                return -ENODEV;
 943
 944        bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
 945        if (!bridge)
 946                return -ENOMEM;
 947
 948        rockchip = pci_host_bridge_priv(bridge);
 949
 950        platform_set_drvdata(pdev, rockchip);
 951        rockchip->dev = dev;
 952        rockchip->is_rc = true;
 953
 954        err = rockchip_pcie_parse_host_dt(rockchip);
 955        if (err)
 956                return err;
 957
 958        err = rockchip_pcie_enable_clocks(rockchip);
 959        if (err)
 960                return err;
 961
 962        err = rockchip_pcie_set_vpcie(rockchip);
 963        if (err) {
 964                dev_err(dev, "failed to set vpcie regulator\n");
 965                goto err_set_vpcie;
 966        }
 967
 968        err = rockchip_pcie_host_init_port(rockchip);
 969        if (err)
 970                goto err_vpcie;
 971
 972        err = rockchip_pcie_init_irq_domain(rockchip);
 973        if (err < 0)
 974                goto err_deinit_port;
 975
 976        err = rockchip_pcie_cfg_atu(rockchip);
 977        if (err)
 978                goto err_remove_irq_domain;
 979
 980        rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
 981        if (!rockchip->msg_region) {
 982                err = -ENOMEM;
 983                goto err_remove_irq_domain;
 984        }
 985
 986        bridge->sysdata = rockchip;
 987        bridge->ops = &rockchip_pcie_ops;
 988
 989        err = rockchip_pcie_setup_irq(rockchip);
 990        if (err)
 991                goto err_remove_irq_domain;
 992
 993        rockchip_pcie_enable_interrupts(rockchip);
 994
 995        err = pci_host_probe(bridge);
 996        if (err < 0)
 997                goto err_remove_irq_domain;
 998
 999        return 0;
1000
1001err_remove_irq_domain:
1002        irq_domain_remove(rockchip->irq_domain);
1003err_deinit_port:
1004        rockchip_pcie_deinit_phys(rockchip);
1005err_vpcie:
1006        if (!IS_ERR(rockchip->vpcie12v))
1007                regulator_disable(rockchip->vpcie12v);
1008        if (!IS_ERR(rockchip->vpcie3v3))
1009                regulator_disable(rockchip->vpcie3v3);
1010        regulator_disable(rockchip->vpcie1v8);
1011        regulator_disable(rockchip->vpcie0v9);
1012err_set_vpcie:
1013        rockchip_pcie_disable_clocks(rockchip);
1014        return err;
1015}
1016
1017static int rockchip_pcie_remove(struct platform_device *pdev)
1018{
1019        struct device *dev = &pdev->dev;
1020        struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1021        struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
1022
1023        pci_stop_root_bus(bridge->bus);
1024        pci_remove_root_bus(bridge->bus);
1025        irq_domain_remove(rockchip->irq_domain);
1026
1027        rockchip_pcie_deinit_phys(rockchip);
1028
1029        rockchip_pcie_disable_clocks(rockchip);
1030
1031        if (!IS_ERR(rockchip->vpcie12v))
1032                regulator_disable(rockchip->vpcie12v);
1033        if (!IS_ERR(rockchip->vpcie3v3))
1034                regulator_disable(rockchip->vpcie3v3);
1035        regulator_disable(rockchip->vpcie1v8);
1036        regulator_disable(rockchip->vpcie0v9);
1037
1038        return 0;
1039}
1040
1041static const struct dev_pm_ops rockchip_pcie_pm_ops = {
1042        SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
1043                                      rockchip_pcie_resume_noirq)
1044};
1045
1046static const struct of_device_id rockchip_pcie_of_match[] = {
1047        { .compatible = "rockchip,rk3399-pcie", },
1048        {}
1049};
1050MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
1051
1052static struct platform_driver rockchip_pcie_driver = {
1053        .driver = {
1054                .name = "rockchip-pcie",
1055                .of_match_table = rockchip_pcie_of_match,
1056                .pm = &rockchip_pcie_pm_ops,
1057        },
1058        .probe = rockchip_pcie_probe,
1059        .remove = rockchip_pcie_remove,
1060};
1061module_platform_driver(rockchip_pcie_driver);
1062
1063MODULE_AUTHOR("Rockchip Inc");
1064MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1065MODULE_LICENSE("GPL v2");
1066