linux/drivers/soc/fsl/qbman/qman_ccsr.c
<<
>>
Prefs
   1/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
   2 *
   3 * Redistribution and use in source and binary forms, with or without
   4 * modification, are permitted provided that the following conditions are met:
   5 *     * Redistributions of source code must retain the above copyright
   6 *       notice, this list of conditions and the following disclaimer.
   7 *     * Redistributions in binary form must reproduce the above copyright
   8 *       notice, this list of conditions and the following disclaimer in the
   9 *       documentation and/or other materials provided with the distribution.
  10 *     * Neither the name of Freescale Semiconductor nor the
  11 *       names of its contributors may be used to endorse or promote products
  12 *       derived from this software without specific prior written permission.
  13 *
  14 * ALTERNATIVELY, this software may be distributed under the terms of the
  15 * GNU General Public License ("GPL") as published by the Free Software
  16 * Foundation, either version 2 of that License or (at your option) any
  17 * later version.
  18 *
  19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 */
  30
  31#include "qman_priv.h"
  32
  33u16 qman_ip_rev;
  34EXPORT_SYMBOL(qman_ip_rev);
  35u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1;
  36EXPORT_SYMBOL(qm_channel_pool1);
  37u16 qm_channel_caam = QMAN_CHANNEL_CAAM;
  38EXPORT_SYMBOL(qm_channel_caam);
  39
  40/* Register offsets */
  41#define REG_QCSP_LIO_CFG(n)     (0x0000 + ((n) * 0x10))
  42#define REG_QCSP_IO_CFG(n)      (0x0004 + ((n) * 0x10))
  43#define REG_QCSP_DD_CFG(n)      (0x000c + ((n) * 0x10))
  44#define REG_DD_CFG              0x0200
  45#define REG_DCP_CFG(n)          (0x0300 + ((n) * 0x10))
  46#define REG_DCP_DD_CFG(n)       (0x0304 + ((n) * 0x10))
  47#define REG_DCP_DLM_AVG(n)      (0x030c + ((n) * 0x10))
  48#define REG_PFDR_FPC            0x0400
  49#define REG_PFDR_FP_HEAD        0x0404
  50#define REG_PFDR_FP_TAIL        0x0408
  51#define REG_PFDR_FP_LWIT        0x0410
  52#define REG_PFDR_CFG            0x0414
  53#define REG_SFDR_CFG            0x0500
  54#define REG_SFDR_IN_USE         0x0504
  55#define REG_WQ_CS_CFG(n)        (0x0600 + ((n) * 0x04))
  56#define REG_WQ_DEF_ENC_WQID     0x0630
  57#define REG_WQ_SC_DD_CFG(n)     (0x640 + ((n) * 0x04))
  58#define REG_WQ_PC_DD_CFG(n)     (0x680 + ((n) * 0x04))
  59#define REG_WQ_DC0_DD_CFG(n)    (0x6c0 + ((n) * 0x04))
  60#define REG_WQ_DC1_DD_CFG(n)    (0x700 + ((n) * 0x04))
  61#define REG_WQ_DCn_DD_CFG(n)    (0x6c0 + ((n) * 0x40)) /* n=2,3 */
  62#define REG_CM_CFG              0x0800
  63#define REG_ECSR                0x0a00
  64#define REG_ECIR                0x0a04
  65#define REG_EADR                0x0a08
  66#define REG_ECIR2               0x0a0c
  67#define REG_EDATA(n)            (0x0a10 + ((n) * 0x04))
  68#define REG_SBEC(n)             (0x0a80 + ((n) * 0x04))
  69#define REG_MCR                 0x0b00
  70#define REG_MCP(n)              (0x0b04 + ((n) * 0x04))
  71#define REG_MISC_CFG            0x0be0
  72#define REG_HID_CFG             0x0bf0
  73#define REG_IDLE_STAT           0x0bf4
  74#define REG_IP_REV_1            0x0bf8
  75#define REG_IP_REV_2            0x0bfc
  76#define REG_FQD_BARE            0x0c00
  77#define REG_PFDR_BARE           0x0c20
  78#define REG_offset_BAR          0x0004  /* relative to REG_[FQD|PFDR]_BARE */
  79#define REG_offset_AR           0x0010  /* relative to REG_[FQD|PFDR]_BARE */
  80#define REG_QCSP_BARE           0x0c80
  81#define REG_QCSP_BAR            0x0c84
  82#define REG_CI_SCHED_CFG        0x0d00
  83#define REG_SRCIDR              0x0d04
  84#define REG_LIODNR              0x0d08
  85#define REG_CI_RLM_AVG          0x0d14
  86#define REG_ERR_ISR             0x0e00
  87#define REG_ERR_IER             0x0e04
  88#define REG_REV3_QCSP_LIO_CFG(n)        (0x1000 + ((n) * 0x10))
  89#define REG_REV3_QCSP_IO_CFG(n) (0x1004 + ((n) * 0x10))
  90#define REG_REV3_QCSP_DD_CFG(n) (0x100c + ((n) * 0x10))
  91
  92/* Assists for QMAN_MCR */
  93#define MCR_INIT_PFDR           0x01000000
  94#define MCR_get_rslt(v)         (u8)((v) >> 24)
  95#define MCR_rslt_idle(r)        (!(r) || ((r) >= 0xf0))
  96#define MCR_rslt_ok(r)          ((r) == 0xf0)
  97#define MCR_rslt_eaccess(r)     ((r) == 0xf8)
  98#define MCR_rslt_inval(r)       ((r) == 0xff)
  99
 100/*
 101 * Corenet initiator settings. Stash request queues are 4-deep to match cores
 102 * ability to snarf. Stash priority is 3, other priorities are 2.
 103 */
 104#define QM_CI_SCHED_CFG_SRCCIV          4
 105#define QM_CI_SCHED_CFG_SRQ_W           3
 106#define QM_CI_SCHED_CFG_RW_W            2
 107#define QM_CI_SCHED_CFG_BMAN_W          2
 108/* write SRCCIV enable */
 109#define QM_CI_SCHED_CFG_SRCCIV_EN       BIT(31)
 110
 111/* Follows WQ_CS_CFG0-5 */
 112enum qm_wq_class {
 113        qm_wq_portal = 0,
 114        qm_wq_pool = 1,
 115        qm_wq_fman0 = 2,
 116        qm_wq_fman1 = 3,
 117        qm_wq_caam = 4,
 118        qm_wq_pme = 5,
 119        qm_wq_first = qm_wq_portal,
 120        qm_wq_last = qm_wq_pme
 121};
 122
 123/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */
 124enum qm_memory {
 125        qm_memory_fqd,
 126        qm_memory_pfdr
 127};
 128
 129/* Used by all error interrupt registers except 'inhibit' */
 130#define QM_EIRQ_CIDE    0x20000000      /* Corenet Initiator Data Error */
 131#define QM_EIRQ_CTDE    0x10000000      /* Corenet Target Data Error */
 132#define QM_EIRQ_CITT    0x08000000      /* Corenet Invalid Target Transaction */
 133#define QM_EIRQ_PLWI    0x04000000      /* PFDR Low Watermark */
 134#define QM_EIRQ_MBEI    0x02000000      /* Multi-bit ECC Error */
 135#define QM_EIRQ_SBEI    0x01000000      /* Single-bit ECC Error */
 136#define QM_EIRQ_PEBI    0x00800000      /* PFDR Enqueues Blocked Interrupt */
 137#define QM_EIRQ_IFSI    0x00020000      /* Invalid FQ Flow Control State */
 138#define QM_EIRQ_ICVI    0x00010000      /* Invalid Command Verb */
 139#define QM_EIRQ_IDDI    0x00000800      /* Invalid Dequeue (Direct-connect) */
 140#define QM_EIRQ_IDFI    0x00000400      /* Invalid Dequeue FQ */
 141#define QM_EIRQ_IDSI    0x00000200      /* Invalid Dequeue Source */
 142#define QM_EIRQ_IDQI    0x00000100      /* Invalid Dequeue Queue */
 143#define QM_EIRQ_IECE    0x00000010      /* Invalid Enqueue Configuration */
 144#define QM_EIRQ_IEOI    0x00000008      /* Invalid Enqueue Overflow */
 145#define QM_EIRQ_IESI    0x00000004      /* Invalid Enqueue State */
 146#define QM_EIRQ_IECI    0x00000002      /* Invalid Enqueue Channel */
 147#define QM_EIRQ_IEQI    0x00000001      /* Invalid Enqueue Queue */
 148
 149/* QMAN_ECIR valid error bit */
 150#define PORTAL_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \
 151                         QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \
 152                         QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI)
 153#define FQID_ECSR_ERR   (QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \
 154                         QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \
 155                         QM_EIRQ_IFSI)
 156
 157struct qm_ecir {
 158        u32 info; /* res[30-31], ptyp[29], pnum[24-28], fqid[0-23] */
 159};
 160
 161static bool qm_ecir_is_dcp(const struct qm_ecir *p)
 162{
 163        return p->info & BIT(29);
 164}
 165
 166static int qm_ecir_get_pnum(const struct qm_ecir *p)
 167{
 168        return (p->info >> 24) & 0x1f;
 169}
 170
 171static int qm_ecir_get_fqid(const struct qm_ecir *p)
 172{
 173        return p->info & (BIT(24) - 1);
 174}
 175
 176struct qm_ecir2 {
 177        u32 info; /* ptyp[31], res[10-30], pnum[0-9] */
 178};
 179
 180static bool qm_ecir2_is_dcp(const struct qm_ecir2 *p)
 181{
 182        return p->info & BIT(31);
 183}
 184
 185static int qm_ecir2_get_pnum(const struct qm_ecir2 *p)
 186{
 187        return p->info & (BIT(10) - 1);
 188}
 189
 190struct qm_eadr {
 191        u32 info; /* memid[24-27], eadr[0-11] */
 192                  /* v3: memid[24-28], eadr[0-15] */
 193};
 194
 195static int qm_eadr_get_memid(const struct qm_eadr *p)
 196{
 197        return (p->info >> 24) & 0xf;
 198}
 199
 200static int qm_eadr_get_eadr(const struct qm_eadr *p)
 201{
 202        return p->info & (BIT(12) - 1);
 203}
 204
 205static int qm_eadr_v3_get_memid(const struct qm_eadr *p)
 206{
 207        return (p->info >> 24) & 0x1f;
 208}
 209
 210static int qm_eadr_v3_get_eadr(const struct qm_eadr *p)
 211{
 212        return p->info & (BIT(16) - 1);
 213}
 214
 215struct qman_hwerr_txt {
 216        u32 mask;
 217        const char *txt;
 218};
 219
 220
 221static const struct qman_hwerr_txt qman_hwerr_txts[] = {
 222        { QM_EIRQ_CIDE, "Corenet Initiator Data Error" },
 223        { QM_EIRQ_CTDE, "Corenet Target Data Error" },
 224        { QM_EIRQ_CITT, "Corenet Invalid Target Transaction" },
 225        { QM_EIRQ_PLWI, "PFDR Low Watermark" },
 226        { QM_EIRQ_MBEI, "Multi-bit ECC Error" },
 227        { QM_EIRQ_SBEI, "Single-bit ECC Error" },
 228        { QM_EIRQ_PEBI, "PFDR Enqueues Blocked Interrupt" },
 229        { QM_EIRQ_ICVI, "Invalid Command Verb" },
 230        { QM_EIRQ_IFSI, "Invalid Flow Control State" },
 231        { QM_EIRQ_IDDI, "Invalid Dequeue (Direct-connect)" },
 232        { QM_EIRQ_IDFI, "Invalid Dequeue FQ" },
 233        { QM_EIRQ_IDSI, "Invalid Dequeue Source" },
 234        { QM_EIRQ_IDQI, "Invalid Dequeue Queue" },
 235        { QM_EIRQ_IECE, "Invalid Enqueue Configuration" },
 236        { QM_EIRQ_IEOI, "Invalid Enqueue Overflow" },
 237        { QM_EIRQ_IESI, "Invalid Enqueue State" },
 238        { QM_EIRQ_IECI, "Invalid Enqueue Channel" },
 239        { QM_EIRQ_IEQI, "Invalid Enqueue Queue" },
 240};
 241
 242struct qman_error_info_mdata {
 243        u16 addr_mask;
 244        u16 bits;
 245        const char *txt;
 246};
 247
 248static const struct qman_error_info_mdata error_mdata[] = {
 249        { 0x01FF, 24, "FQD cache tag memory 0" },
 250        { 0x01FF, 24, "FQD cache tag memory 1" },
 251        { 0x01FF, 24, "FQD cache tag memory 2" },
 252        { 0x01FF, 24, "FQD cache tag memory 3" },
 253        { 0x0FFF, 512, "FQD cache memory" },
 254        { 0x07FF, 128, "SFDR memory" },
 255        { 0x01FF, 72, "WQ context memory" },
 256        { 0x00FF, 240, "CGR memory" },
 257        { 0x00FF, 302, "Internal Order Restoration List memory" },
 258        { 0x01FF, 256, "SW portal ring memory" },
 259};
 260
 261#define QMAN_ERRS_TO_DISABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI)
 262
 263/*
 264 * TODO: unimplemented registers
 265 *
 266 * Keeping a list here of QMan registers I have not yet covered;
 267 * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR,
 268 * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG,
 269 * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12
 270 */
 271
 272/* Pointer to the start of the QMan's CCSR space */
 273static u32 __iomem *qm_ccsr_start;
 274/* A SDQCR mask comprising all the available/visible pool channels */
 275static u32 qm_pools_sdqcr;
 276static int __qman_probed;
 277static int  __qman_requires_cleanup;
 278
 279static inline u32 qm_ccsr_in(u32 offset)
 280{
 281        return ioread32be(qm_ccsr_start + offset/4);
 282}
 283
 284static inline void qm_ccsr_out(u32 offset, u32 val)
 285{
 286        iowrite32be(val, qm_ccsr_start + offset/4);
 287}
 288
 289u32 qm_get_pools_sdqcr(void)
 290{
 291        return qm_pools_sdqcr;
 292}
 293
 294enum qm_dc_portal {
 295        qm_dc_portal_fman0 = 0,
 296        qm_dc_portal_fman1 = 1
 297};
 298
 299static void qm_set_dc(enum qm_dc_portal portal, int ed, u8 sernd)
 300{
 301        DPAA_ASSERT(!ed || portal == qm_dc_portal_fman0 ||
 302                    portal == qm_dc_portal_fman1);
 303        if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
 304                qm_ccsr_out(REG_DCP_CFG(portal),
 305                            (ed ? 0x1000 : 0) | (sernd & 0x3ff));
 306        else
 307                qm_ccsr_out(REG_DCP_CFG(portal),
 308                            (ed ? 0x100 : 0) | (sernd & 0x1f));
 309}
 310
 311static void qm_set_wq_scheduling(enum qm_wq_class wq_class,
 312                                 u8 cs_elev, u8 csw2, u8 csw3, u8 csw4,
 313                                 u8 csw5, u8 csw6, u8 csw7)
 314{
 315        qm_ccsr_out(REG_WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) |
 316                    ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) |
 317                    ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) |
 318                    ((csw6 & 0x7) << 4) | (csw7 & 0x7));
 319}
 320
 321static void qm_set_hid(void)
 322{
 323        qm_ccsr_out(REG_HID_CFG, 0);
 324}
 325
 326static void qm_set_corenet_initiator(void)
 327{
 328        qm_ccsr_out(REG_CI_SCHED_CFG, QM_CI_SCHED_CFG_SRCCIV_EN |
 329                    (QM_CI_SCHED_CFG_SRCCIV << 24) |
 330                    (QM_CI_SCHED_CFG_SRQ_W << 8) |
 331                    (QM_CI_SCHED_CFG_RW_W << 4) |
 332                    QM_CI_SCHED_CFG_BMAN_W);
 333}
 334
 335static void qm_get_version(u16 *id, u8 *major, u8 *minor)
 336{
 337        u32 v = qm_ccsr_in(REG_IP_REV_1);
 338        *id = (v >> 16);
 339        *major = (v >> 8) & 0xff;
 340        *minor = v & 0xff;
 341}
 342
 343#define PFDR_AR_EN              BIT(31)
 344static int qm_set_memory(enum qm_memory memory, u64 ba, u32 size)
 345{
 346        void *ptr;
 347        u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE;
 348        u32 exp = ilog2(size);
 349        u32 bar, bare;
 350
 351        /* choke if size isn't within range */
 352        DPAA_ASSERT((size >= 4096) && (size <= 1024*1024*1024) &&
 353                    is_power_of_2(size));
 354        /* choke if 'ba' has lower-alignment than 'size' */
 355        DPAA_ASSERT(!(ba & (size - 1)));
 356
 357        /* Check to see if QMan has already been initialized */
 358        bar = qm_ccsr_in(offset + REG_offset_BAR);
 359        if (bar) {
 360                /* Maker sure ba == what was programmed) */
 361                bare = qm_ccsr_in(offset);
 362                if (bare != upper_32_bits(ba) || bar != lower_32_bits(ba)) {
 363                        pr_err("Attempted to reinitialize QMan with different BAR, got 0x%llx read BARE=0x%x BAR=0x%x\n",
 364                               ba, bare, bar);
 365                        return -ENOMEM;
 366                }
 367                __qman_requires_cleanup = 1;
 368                /* Return 1 to indicate memory was previously programmed */
 369                return 1;
 370        }
 371        /* Need to temporarily map the area to make sure it is zeroed */
 372        ptr = memremap(ba, size, MEMREMAP_WB);
 373        if (!ptr) {
 374                pr_crit("memremap() of QMan private memory failed\n");
 375                return -ENOMEM;
 376        }
 377        memset(ptr, 0, size);
 378
 379#ifdef CONFIG_PPC
 380        /*
 381         * PPC doesn't appear to flush the cache on memunmap() but the
 382         * cache must be flushed since QMan does non coherent accesses
 383         * to this memory
 384         */
 385        flush_dcache_range((unsigned long) ptr, (unsigned long) ptr+size);
 386#endif
 387        memunmap(ptr);
 388
 389        qm_ccsr_out(offset, upper_32_bits(ba));
 390        qm_ccsr_out(offset + REG_offset_BAR, lower_32_bits(ba));
 391        qm_ccsr_out(offset + REG_offset_AR, PFDR_AR_EN | (exp - 1));
 392        return 0;
 393}
 394
 395static void qm_set_pfdr_threshold(u32 th, u8 k)
 396{
 397        qm_ccsr_out(REG_PFDR_FP_LWIT, th & 0xffffff);
 398        qm_ccsr_out(REG_PFDR_CFG, k);
 399}
 400
 401static void qm_set_sfdr_threshold(u16 th)
 402{
 403        qm_ccsr_out(REG_SFDR_CFG, th & 0x3ff);
 404}
 405
 406static int qm_init_pfdr(struct device *dev, u32 pfdr_start, u32 num)
 407{
 408        u8 rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR));
 409
 410        DPAA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num);
 411        /* Make sure the command interface is 'idle' */
 412        if (!MCR_rslt_idle(rslt)) {
 413                dev_crit(dev, "QMAN_MCR isn't idle");
 414                WARN_ON(1);
 415        }
 416
 417        /* Write the MCR command params then the verb */
 418        qm_ccsr_out(REG_MCP(0), pfdr_start);
 419        /*
 420         * TODO: remove this - it's a workaround for a model bug that is
 421         * corrected in more recent versions. We use the workaround until
 422         * everyone has upgraded.
 423         */
 424        qm_ccsr_out(REG_MCP(1), pfdr_start + num - 16);
 425        dma_wmb();
 426        qm_ccsr_out(REG_MCR, MCR_INIT_PFDR);
 427        /* Poll for the result */
 428        do {
 429                rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR));
 430        } while (!MCR_rslt_idle(rslt));
 431        if (MCR_rslt_ok(rslt))
 432                return 0;
 433        if (MCR_rslt_eaccess(rslt))
 434                return -EACCES;
 435        if (MCR_rslt_inval(rslt))
 436                return -EINVAL;
 437        dev_crit(dev, "Unexpected result from MCR_INIT_PFDR: %02x\n", rslt);
 438        return -ENODEV;
 439}
 440
 441/*
 442 * QMan needs two global memory areas initialized at boot time:
 443 *  1) FQD: Frame Queue Descriptors used to manage frame queues
 444 *  2) PFDR: Packed Frame Queue Descriptor Records used to store frames
 445 * Both areas are reserved using the device tree reserved memory framework
 446 * and the addresses and sizes are initialized when the QMan device is probed
 447 */
 448static dma_addr_t fqd_a, pfdr_a;
 449static size_t fqd_sz, pfdr_sz;
 450
 451#ifdef CONFIG_PPC
 452/*
 453 * Support for PPC Device Tree backward compatibility when compatible
 454 * string is set to fsl-qman-fqd and fsl-qman-pfdr
 455 */
 456static int zero_priv_mem(phys_addr_t addr, size_t sz)
 457{
 458        /* map as cacheable, non-guarded */
 459        void __iomem *tmpp = ioremap_cache(addr, sz);
 460
 461        if (!tmpp)
 462                return -ENOMEM;
 463
 464        memset_io(tmpp, 0, sz);
 465        flush_dcache_range((unsigned long)tmpp,
 466                           (unsigned long)tmpp + sz);
 467        iounmap(tmpp);
 468
 469        return 0;
 470}
 471
 472static int qman_fqd(struct reserved_mem *rmem)
 473{
 474        fqd_a = rmem->base;
 475        fqd_sz = rmem->size;
 476
 477        WARN_ON(!(fqd_a && fqd_sz));
 478        return 0;
 479}
 480RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd);
 481
 482static int qman_pfdr(struct reserved_mem *rmem)
 483{
 484        pfdr_a = rmem->base;
 485        pfdr_sz = rmem->size;
 486
 487        WARN_ON(!(pfdr_a && pfdr_sz));
 488
 489        return 0;
 490}
 491RESERVEDMEM_OF_DECLARE(qman_pfdr, "fsl,qman-pfdr", qman_pfdr);
 492
 493#endif
 494
 495unsigned int qm_get_fqid_maxcnt(void)
 496{
 497        return fqd_sz / 64;
 498}
 499
 500static void log_edata_bits(struct device *dev, u32 bit_count)
 501{
 502        u32 i, j, mask = 0xffffffff;
 503
 504        dev_warn(dev, "ErrInt, EDATA:\n");
 505        i = bit_count / 32;
 506        if (bit_count % 32) {
 507                i++;
 508                mask = ~(mask << bit_count % 32);
 509        }
 510        j = 16 - i;
 511        dev_warn(dev, "  0x%08x\n", qm_ccsr_in(REG_EDATA(j)) & mask);
 512        j++;
 513        for (; j < 16; j++)
 514                dev_warn(dev, "  0x%08x\n", qm_ccsr_in(REG_EDATA(j)));
 515}
 516
 517static void log_additional_error_info(struct device *dev, u32 isr_val,
 518                                      u32 ecsr_val)
 519{
 520        struct qm_ecir ecir_val;
 521        struct qm_eadr eadr_val;
 522        int memid;
 523
 524        ecir_val.info = qm_ccsr_in(REG_ECIR);
 525        /* Is portal info valid */
 526        if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
 527                struct qm_ecir2 ecir2_val;
 528
 529                ecir2_val.info = qm_ccsr_in(REG_ECIR2);
 530                if (ecsr_val & PORTAL_ECSR_ERR) {
 531                        dev_warn(dev, "ErrInt: %s id %d\n",
 532                                 qm_ecir2_is_dcp(&ecir2_val) ? "DCP" : "SWP",
 533                                 qm_ecir2_get_pnum(&ecir2_val));
 534                }
 535                if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE))
 536                        dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n",
 537                                 qm_ecir_get_fqid(&ecir_val));
 538
 539                if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
 540                        eadr_val.info = qm_ccsr_in(REG_EADR);
 541                        memid = qm_eadr_v3_get_memid(&eadr_val);
 542                        dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n",
 543                                 error_mdata[memid].txt,
 544                                 error_mdata[memid].addr_mask
 545                                        & qm_eadr_v3_get_eadr(&eadr_val));
 546                        log_edata_bits(dev, error_mdata[memid].bits);
 547                }
 548        } else {
 549                if (ecsr_val & PORTAL_ECSR_ERR) {
 550                        dev_warn(dev, "ErrInt: %s id %d\n",
 551                                 qm_ecir_is_dcp(&ecir_val) ? "DCP" : "SWP",
 552                                 qm_ecir_get_pnum(&ecir_val));
 553                }
 554                if (ecsr_val & FQID_ECSR_ERR)
 555                        dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n",
 556                                 qm_ecir_get_fqid(&ecir_val));
 557
 558                if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
 559                        eadr_val.info = qm_ccsr_in(REG_EADR);
 560                        memid = qm_eadr_get_memid(&eadr_val);
 561                        dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n",
 562                                 error_mdata[memid].txt,
 563                                 error_mdata[memid].addr_mask
 564                                        & qm_eadr_get_eadr(&eadr_val));
 565                        log_edata_bits(dev, error_mdata[memid].bits);
 566                }
 567        }
 568}
 569
 570static irqreturn_t qman_isr(int irq, void *ptr)
 571{
 572        u32 isr_val, ier_val, ecsr_val, isr_mask, i;
 573        struct device *dev = ptr;
 574
 575        ier_val = qm_ccsr_in(REG_ERR_IER);
 576        isr_val = qm_ccsr_in(REG_ERR_ISR);
 577        ecsr_val = qm_ccsr_in(REG_ECSR);
 578        isr_mask = isr_val & ier_val;
 579
 580        if (!isr_mask)
 581                return IRQ_NONE;
 582
 583        for (i = 0; i < ARRAY_SIZE(qman_hwerr_txts); i++) {
 584                if (qman_hwerr_txts[i].mask & isr_mask) {
 585                        dev_err_ratelimited(dev, "ErrInt: %s\n",
 586                                            qman_hwerr_txts[i].txt);
 587                        if (qman_hwerr_txts[i].mask & ecsr_val) {
 588                                log_additional_error_info(dev, isr_mask,
 589                                                          ecsr_val);
 590                                /* Re-arm error capture registers */
 591                                qm_ccsr_out(REG_ECSR, ecsr_val);
 592                        }
 593                        if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_DISABLE) {
 594                                dev_dbg(dev, "Disabling error 0x%x\n",
 595                                        qman_hwerr_txts[i].mask);
 596                                ier_val &= ~qman_hwerr_txts[i].mask;
 597                                qm_ccsr_out(REG_ERR_IER, ier_val);
 598                        }
 599                }
 600        }
 601        qm_ccsr_out(REG_ERR_ISR, isr_val);
 602
 603        return IRQ_HANDLED;
 604}
 605
 606static int qman_init_ccsr(struct device *dev)
 607{
 608        int i, err;
 609
 610        /* FQD memory */
 611        err = qm_set_memory(qm_memory_fqd, fqd_a, fqd_sz);
 612        if (err < 0)
 613                return err;
 614        /* PFDR memory */
 615        err = qm_set_memory(qm_memory_pfdr, pfdr_a, pfdr_sz);
 616        if (err < 0)
 617                return err;
 618        /* Only initialize PFDRs if the QMan was not initialized before */
 619        if (err == 0) {
 620                err = qm_init_pfdr(dev, 8, pfdr_sz / 64 - 8);
 621                if (err)
 622                        return err;
 623        }
 624        /* thresholds */
 625        qm_set_pfdr_threshold(512, 64);
 626        qm_set_sfdr_threshold(128);
 627        /* clear stale PEBI bit from interrupt status register */
 628        qm_ccsr_out(REG_ERR_ISR, QM_EIRQ_PEBI);
 629        /* corenet initiator settings */
 630        qm_set_corenet_initiator();
 631        /* HID settings */
 632        qm_set_hid();
 633        /* Set scheduling weights to defaults */
 634        for (i = qm_wq_first; i <= qm_wq_last; i++)
 635                qm_set_wq_scheduling(i, 0, 0, 0, 0, 0, 0, 0);
 636        /* We are not prepared to accept ERNs for hardware enqueues */
 637        qm_set_dc(qm_dc_portal_fman0, 1, 0);
 638        qm_set_dc(qm_dc_portal_fman1, 1, 0);
 639        return 0;
 640}
 641
 642#define LIO_CFG_LIODN_MASK 0x0fff0000
 643void __qman_liodn_fixup(u16 channel)
 644{
 645        static int done;
 646        static u32 liodn_offset;
 647        u32 before, after;
 648        int idx = channel - QM_CHANNEL_SWPORTAL0;
 649
 650        if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
 651                before = qm_ccsr_in(REG_REV3_QCSP_LIO_CFG(idx));
 652        else
 653                before = qm_ccsr_in(REG_QCSP_LIO_CFG(idx));
 654        if (!done) {
 655                liodn_offset = before & LIO_CFG_LIODN_MASK;
 656                done = 1;
 657                return;
 658        }
 659        after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset;
 660        if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
 661                qm_ccsr_out(REG_REV3_QCSP_LIO_CFG(idx), after);
 662        else
 663                qm_ccsr_out(REG_QCSP_LIO_CFG(idx), after);
 664}
 665
 666#define IO_CFG_SDEST_MASK 0x00ff0000
 667void qman_set_sdest(u16 channel, unsigned int cpu_idx)
 668{
 669        int idx = channel - QM_CHANNEL_SWPORTAL0;
 670        u32 before, after;
 671
 672        if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
 673                before = qm_ccsr_in(REG_REV3_QCSP_IO_CFG(idx));
 674                /* Each pair of vcpu share the same SRQ(SDEST) */
 675                cpu_idx /= 2;
 676                after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
 677                qm_ccsr_out(REG_REV3_QCSP_IO_CFG(idx), after);
 678        } else {
 679                before = qm_ccsr_in(REG_QCSP_IO_CFG(idx));
 680                after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
 681                qm_ccsr_out(REG_QCSP_IO_CFG(idx), after);
 682        }
 683}
 684
 685static int qman_resource_init(struct device *dev)
 686{
 687        int pool_chan_num, cgrid_num;
 688        int ret, i;
 689
 690        switch (qman_ip_rev >> 8) {
 691        case 1:
 692                pool_chan_num = 15;
 693                cgrid_num = 256;
 694                break;
 695        case 2:
 696                pool_chan_num = 3;
 697                cgrid_num = 64;
 698                break;
 699        case 3:
 700                pool_chan_num = 15;
 701                cgrid_num = 256;
 702                break;
 703        default:
 704                return -ENODEV;
 705        }
 706
 707        ret = gen_pool_add(qm_qpalloc, qm_channel_pool1 | DPAA_GENALLOC_OFF,
 708                           pool_chan_num, -1);
 709        if (ret) {
 710                dev_err(dev, "Failed to seed pool channels (%d)\n", ret);
 711                return ret;
 712        }
 713
 714        ret = gen_pool_add(qm_cgralloc, DPAA_GENALLOC_OFF, cgrid_num, -1);
 715        if (ret) {
 716                dev_err(dev, "Failed to seed CGRID range (%d)\n", ret);
 717                return ret;
 718        }
 719
 720        /* parse pool channels into the SDQCR mask */
 721        for (i = 0; i < cgrid_num; i++)
 722                qm_pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(i);
 723
 724        ret = gen_pool_add(qm_fqalloc, QM_FQID_RANGE_START | DPAA_GENALLOC_OFF,
 725                           qm_get_fqid_maxcnt() - QM_FQID_RANGE_START, -1);
 726        if (ret) {
 727                dev_err(dev, "Failed to seed FQID range (%d)\n", ret);
 728                return ret;
 729        }
 730
 731        return 0;
 732}
 733
 734int qman_is_probed(void)
 735{
 736        return __qman_probed;
 737}
 738EXPORT_SYMBOL_GPL(qman_is_probed);
 739
 740int qman_requires_cleanup(void)
 741{
 742        return __qman_requires_cleanup;
 743}
 744
 745void qman_done_cleanup(void)
 746{
 747        qman_enable_irqs();
 748        __qman_requires_cleanup = 0;
 749}
 750
 751
 752static int fsl_qman_probe(struct platform_device *pdev)
 753{
 754        struct device *dev = &pdev->dev;
 755        struct device_node *node = dev->of_node;
 756        struct resource *res;
 757        int ret, err_irq;
 758        u16 id;
 759        u8 major, minor;
 760
 761        __qman_probed = -1;
 762
 763        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 764        if (!res) {
 765                dev_err(dev, "Can't get %pOF property 'IORESOURCE_MEM'\n",
 766                        node);
 767                return -ENXIO;
 768        }
 769        qm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res));
 770        if (!qm_ccsr_start)
 771                return -ENXIO;
 772
 773        qm_get_version(&id, &major, &minor);
 774        if (major == 1 && minor == 0) {
 775                dev_err(dev, "Rev1.0 on P4080 rev1 is not supported!\n");
 776                        return -ENODEV;
 777        } else if (major == 1 && minor == 1)
 778                qman_ip_rev = QMAN_REV11;
 779        else if (major == 1 && minor == 2)
 780                qman_ip_rev = QMAN_REV12;
 781        else if (major == 2 && minor == 0)
 782                qman_ip_rev = QMAN_REV20;
 783        else if (major == 3 && minor == 0)
 784                qman_ip_rev = QMAN_REV30;
 785        else if (major == 3 && minor == 1)
 786                qman_ip_rev = QMAN_REV31;
 787        else if (major == 3 && minor == 2)
 788                qman_ip_rev = QMAN_REV32;
 789        else {
 790                dev_err(dev, "Unknown QMan version\n");
 791                return -ENODEV;
 792        }
 793
 794        if ((qman_ip_rev & 0xff00) >= QMAN_REV30) {
 795                qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3;
 796                qm_channel_caam = QMAN_CHANNEL_CAAM_REV3;
 797        }
 798
 799        if (fqd_a) {
 800#ifdef CONFIG_PPC
 801                /*
 802                 * For PPC backward DT compatibility
 803                 * FQD memory MUST be zero'd by software
 804                 */
 805                zero_priv_mem(fqd_a, fqd_sz);
 806#else
 807                WARN(1, "Unexpected architecture using non shared-dma-mem reservations");
 808#endif
 809        } else {
 810                /*
 811                 * Order of memory regions is assumed as FQD followed by PFDR
 812                 * in order to ensure allocations from the correct regions the
 813                 * driver initializes then allocates each piece in order
 814                 */
 815                ret = qbman_init_private_mem(dev, 0, &fqd_a, &fqd_sz);
 816                if (ret) {
 817                        dev_err(dev, "qbman_init_private_mem() for FQD failed 0x%x\n",
 818                                ret);
 819                        return -ENODEV;
 820                }
 821        }
 822        dev_dbg(dev, "Allocated FQD 0x%llx 0x%zx\n", fqd_a, fqd_sz);
 823
 824        if (!pfdr_a) {
 825                /* Setup PFDR memory */
 826                ret = qbman_init_private_mem(dev, 1, &pfdr_a, &pfdr_sz);
 827                if (ret) {
 828                        dev_err(dev, "qbman_init_private_mem() for PFDR failed 0x%x\n",
 829                                ret);
 830                        return -ENODEV;
 831                }
 832        }
 833        dev_dbg(dev, "Allocated PFDR 0x%llx 0x%zx\n", pfdr_a, pfdr_sz);
 834
 835        ret = qman_init_ccsr(dev);
 836        if (ret) {
 837                dev_err(dev, "CCSR setup failed\n");
 838                return ret;
 839        }
 840
 841        err_irq = platform_get_irq(pdev, 0);
 842        if (err_irq <= 0) {
 843                dev_info(dev, "Can't get %pOF property 'interrupts'\n",
 844                         node);
 845                return -ENODEV;
 846        }
 847        ret = devm_request_irq(dev, err_irq, qman_isr, IRQF_SHARED, "qman-err",
 848                               dev);
 849        if (ret)  {
 850                dev_err(dev, "devm_request_irq() failed %d for '%pOF'\n",
 851                        ret, node);
 852                return ret;
 853        }
 854
 855        /*
 856         * Write-to-clear any stale bits, (eg. starvation being asserted prior
 857         * to resource allocation during driver init).
 858         */
 859        qm_ccsr_out(REG_ERR_ISR, 0xffffffff);
 860        /* Enable Error Interrupts */
 861        qm_ccsr_out(REG_ERR_IER, 0xffffffff);
 862
 863        qm_fqalloc = devm_gen_pool_create(dev, 0, -1, "qman-fqalloc");
 864        if (IS_ERR(qm_fqalloc)) {
 865                ret = PTR_ERR(qm_fqalloc);
 866                dev_err(dev, "qman-fqalloc pool init failed (%d)\n", ret);
 867                return ret;
 868        }
 869
 870        qm_qpalloc = devm_gen_pool_create(dev, 0, -1, "qman-qpalloc");
 871        if (IS_ERR(qm_qpalloc)) {
 872                ret = PTR_ERR(qm_qpalloc);
 873                dev_err(dev, "qman-qpalloc pool init failed (%d)\n", ret);
 874                return ret;
 875        }
 876
 877        qm_cgralloc = devm_gen_pool_create(dev, 0, -1, "qman-cgralloc");
 878        if (IS_ERR(qm_cgralloc)) {
 879                ret = PTR_ERR(qm_cgralloc);
 880                dev_err(dev, "qman-cgralloc pool init failed (%d)\n", ret);
 881                return ret;
 882        }
 883
 884        ret = qman_resource_init(dev);
 885        if (ret)
 886                return ret;
 887
 888        ret = qman_alloc_fq_table(qm_get_fqid_maxcnt());
 889        if (ret)
 890                return ret;
 891
 892        ret = qman_wq_alloc();
 893        if (ret)
 894                return ret;
 895
 896        __qman_probed = 1;
 897
 898        return 0;
 899}
 900
 901static const struct of_device_id fsl_qman_ids[] = {
 902        {
 903                .compatible = "fsl,qman",
 904        },
 905        {}
 906};
 907
 908static struct platform_driver fsl_qman_driver = {
 909        .driver = {
 910                .name = KBUILD_MODNAME,
 911                .of_match_table = fsl_qman_ids,
 912                .suppress_bind_attrs = true,
 913        },
 914        .probe = fsl_qman_probe,
 915};
 916
 917builtin_platform_driver(fsl_qman_driver);
 918
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.