linux/drivers/edac/thunderx_edac.c
<<
>>
Prefs
   1/*
   2 * Cavium ThunderX memory controller kernel module
   3 *
   4 * This file is subject to the terms and conditions of the GNU General Public
   5 * License.  See the file "COPYING" in the main directory of this archive
   6 * for more details.
   7 *
   8 * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
   9 *
  10 */
  11
  12#include <linux/module.h>
  13#include <linux/pci.h>
  14#include <linux/edac.h>
  15#include <linux/interrupt.h>
  16#include <linux/string.h>
  17#include <linux/stop_machine.h>
  18#include <linux/delay.h>
  19#include <linux/sizes.h>
  20#include <linux/atomic.h>
  21#include <linux/bitfield.h>
  22#include <linux/circ_buf.h>
  23
  24#include <asm/page.h>
  25
  26#include "edac_module.h"
  27
  28#define phys_to_pfn(phys)       (PFN_DOWN(phys))
  29
  30#define THUNDERX_NODE           GENMASK(45, 44)
  31
  32enum {
  33        ERR_CORRECTED   = 1,
  34        ERR_UNCORRECTED = 2,
  35        ERR_UNKNOWN     = 3,
  36};
  37
  38#define MAX_SYNDROME_REGS 4
  39
  40struct error_syndrome {
  41        u64 reg[MAX_SYNDROME_REGS];
  42};
  43
  44struct error_descr {
  45        int     type;
  46        u64     mask;
  47        char    *descr;
  48};
  49
  50static void decode_register(char *str, size_t size,
  51                           const struct error_descr *descr,
  52                           const uint64_t reg)
  53{
  54        int ret = 0;
  55
  56        while (descr->type && descr->mask && descr->descr) {
  57                if (reg & descr->mask) {
  58                        ret = snprintf(str, size, "\n\t%s, %s",
  59                                       descr->type == ERR_CORRECTED ?
  60                                         "Corrected" : "Uncorrected",
  61                                       descr->descr);
  62                        str += ret;
  63                        size -= ret;
  64                }
  65                descr++;
  66        }
  67}
  68
  69static unsigned long get_bits(unsigned long data, int pos, int width)
  70{
  71        return (data >> pos) & ((1 << width) - 1);
  72}
  73
  74#define L2C_CTL                 0x87E080800000
  75#define L2C_CTL_DISIDXALIAS     BIT(0)
  76
  77#define PCI_DEVICE_ID_THUNDER_LMC 0xa022
  78
  79#define LMC_FADR                0x20
  80#define LMC_FADR_FDIMM(x)       ((x >> 37) & 0x1)
  81#define LMC_FADR_FBUNK(x)       ((x >> 36) & 0x1)
  82#define LMC_FADR_FBANK(x)       ((x >> 32) & 0xf)
  83#define LMC_FADR_FROW(x)        ((x >> 14) & 0xffff)
  84#define LMC_FADR_FCOL(x)        ((x >> 0) & 0x1fff)
  85
  86#define LMC_NXM_FADR            0x28
  87#define LMC_ECC_SYND            0x38
  88
  89#define LMC_ECC_PARITY_TEST     0x108
  90
  91#define LMC_INT_W1S             0x150
  92
  93#define LMC_INT_ENA_W1C         0x158
  94#define LMC_INT_ENA_W1S         0x160
  95
  96#define LMC_CONFIG              0x188
  97
  98#define LMC_CONFIG_BG2          BIT(62)
  99#define LMC_CONFIG_RANK_ENA     BIT(42)
 100#define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
 101#define LMC_CONFIG_ROW_LSB(x)   (((x) >> 2) & 0x7)
 102
 103#define LMC_CONTROL             0x190
 104#define LMC_CONTROL_XOR_BANK    BIT(16)
 105
 106#define LMC_INT                 0x1F0
 107
 108#define LMC_INT_DDR_ERR         BIT(11)
 109#define LMC_INT_DED_ERR         (0xFUL << 5)
 110#define LMC_INT_SEC_ERR         (0xFUL << 1)
 111#define LMC_INT_NXM_WR_MASK     BIT(0)
 112
 113#define LMC_DDR_PLL_CTL         0x258
 114#define LMC_DDR_PLL_CTL_DDR4    BIT(29)
 115
 116#define LMC_FADR_SCRAMBLED      0x330
 117
 118#define LMC_INT_UE              (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
 119                                 LMC_INT_NXM_WR_MASK)
 120
 121#define LMC_INT_CE              (LMC_INT_SEC_ERR)
 122
 123static const struct error_descr lmc_errors[] = {
 124        {
 125                .type  = ERR_CORRECTED,
 126                .mask  = LMC_INT_SEC_ERR,
 127                .descr = "Single-bit ECC error",
 128        },
 129        {
 130                .type  = ERR_UNCORRECTED,
 131                .mask  = LMC_INT_DDR_ERR,
 132                .descr = "DDR chip error",
 133        },
 134        {
 135                .type  = ERR_UNCORRECTED,
 136                .mask  = LMC_INT_DED_ERR,
 137                .descr = "Double-bit ECC error",
 138        },
 139        {
 140                .type = ERR_UNCORRECTED,
 141                .mask = LMC_INT_NXM_WR_MASK,
 142                .descr = "Non-existent memory write",
 143        },
 144        {0, 0, NULL},
 145};
 146
 147#define LMC_INT_EN_DDR_ERROR_ALERT_ENA  BIT(5)
 148#define LMC_INT_EN_DLCRAM_DED_ERR       BIT(4)
 149#define LMC_INT_EN_DLCRAM_SEC_ERR       BIT(3)
 150#define LMC_INT_INTR_DED_ENA            BIT(2)
 151#define LMC_INT_INTR_SEC_ENA            BIT(1)
 152#define LMC_INT_INTR_NXM_WR_ENA         BIT(0)
 153
 154#define LMC_INT_ENA_ALL                 GENMASK(5, 0)
 155
 156#define LMC_DDR_PLL_CTL         0x258
 157#define LMC_DDR_PLL_CTL_DDR4    BIT(29)
 158
 159#define LMC_CONTROL             0x190
 160#define LMC_CONTROL_RDIMM       BIT(0)
 161
 162#define LMC_SCRAM_FADR          0x330
 163
 164#define LMC_CHAR_MASK0          0x228
 165#define LMC_CHAR_MASK2          0x238
 166
 167#define RING_ENTRIES    8
 168
 169struct debugfs_entry {
 170        const char *name;
 171        umode_t mode;
 172        const struct file_operations fops;
 173};
 174
 175struct lmc_err_ctx {
 176        u64 reg_int;
 177        u64 reg_fadr;
 178        u64 reg_nxm_fadr;
 179        u64 reg_scram_fadr;
 180        u64 reg_ecc_synd;
 181};
 182
 183struct thunderx_lmc {
 184        void __iomem *regs;
 185        struct pci_dev *pdev;
 186        struct msix_entry msix_ent;
 187
 188        atomic_t ecc_int;
 189
 190        u64 mask0;
 191        u64 mask2;
 192        u64 parity_test;
 193        u64 node;
 194
 195        int xbits;
 196        int bank_width;
 197        int pbank_lsb;
 198        int dimm_lsb;
 199        int rank_lsb;
 200        int bank_lsb;
 201        int row_lsb;
 202        int col_hi_lsb;
 203
 204        int xor_bank;
 205        int l2c_alias;
 206
 207        struct page *mem;
 208
 209        struct lmc_err_ctx err_ctx[RING_ENTRIES];
 210        unsigned long ring_head;
 211        unsigned long ring_tail;
 212};
 213
 214#define ring_pos(pos, size) ((pos) & (size - 1))
 215
 216#define DEBUGFS_STRUCT(_name, _mode, _write, _read)                         \
 217static struct debugfs_entry debugfs_##_name = {                             \
 218        .name = __stringify(_name),                                         \
 219        .mode = VERIFY_OCTAL_PERMISSIONS(_mode),                            \
 220        .fops = {                                                           \
 221                .open = simple_open,                                        \
 222                .write = _write,                                            \
 223                .read  = _read,                                             \
 224                .llseek = generic_file_llseek,                              \
 225        },                                                                  \
 226}
 227
 228#define DEBUGFS_FIELD_ATTR(_type, _field)                                   \
 229static ssize_t thunderx_##_type##_##_field##_read(struct file *file,        \
 230                                            char __user *data,              \
 231                                            size_t count, loff_t *ppos)     \
 232{                                                                           \
 233        struct thunderx_##_type *pdata = file->private_data;                \
 234        char buf[20];                                                       \
 235                                                                            \
 236        snprintf(buf, count, "0x%016llx", pdata->_field);                   \
 237        return simple_read_from_buffer(data, count, ppos,                   \
 238                                       buf, sizeof(buf));                   \
 239}                                                                           \
 240                                                                            \
 241static ssize_t thunderx_##_type##_##_field##_write(struct file *file,       \
 242                                             const char __user *data,       \
 243                                             size_t count, loff_t *ppos)    \
 244{                                                                           \
 245        struct thunderx_##_type *pdata = file->private_data;                \
 246        int res;                                                            \
 247                                                                            \
 248        res = kstrtoull_from_user(data, count, 0, &pdata->_field);          \
 249                                                                            \
 250        return res ? res : count;                                           \
 251}                                                                           \
 252                                                                            \
 253DEBUGFS_STRUCT(_field, 0600,                                                \
 254                   thunderx_##_type##_##_field##_write,                     \
 255                   thunderx_##_type##_##_field##_read)                      \
 256
 257#define DEBUGFS_REG_ATTR(_type, _name, _reg)                                \
 258static ssize_t thunderx_##_type##_##_name##_read(struct file *file,         \
 259                                           char __user *data,               \
 260                                           size_t count, loff_t *ppos)      \
 261{                                                                           \
 262        struct thunderx_##_type *pdata = file->private_data;                \
 263        char buf[20];                                                       \
 264                                                                            \
 265        sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));               \
 266        return simple_read_from_buffer(data, count, ppos,                   \
 267                                       buf, sizeof(buf));                   \
 268}                                                                           \
 269                                                                            \
 270static ssize_t thunderx_##_type##_##_name##_write(struct file *file,        \
 271                                            const char __user *data,        \
 272                                            size_t count, loff_t *ppos)     \
 273{                                                                           \
 274        struct thunderx_##_type *pdata = file->private_data;                \
 275        u64 val;                                                            \
 276        int res;                                                            \
 277                                                                            \
 278        res = kstrtoull_from_user(data, count, 0, &val);                    \
 279                                                                            \
 280        if (!res) {                                                         \
 281                writeq(val, pdata->regs + _reg);                            \
 282                res = count;                                                \
 283        }                                                                   \
 284                                                                            \
 285        return res;                                                         \
 286}                                                                           \
 287                                                                            \
 288DEBUGFS_STRUCT(_name, 0600,                                                 \
 289               thunderx_##_type##_##_name##_write,                          \
 290               thunderx_##_type##_##_name##_read)
 291
 292#define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
 293
 294/*
 295 * To get an ECC error injected, the following steps are needed:
 296 * - Setup the ECC injection by writing the appropriate parameters:
 297 *      echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
 298 *      echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
 299 *      echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
 300 * - Do the actual injection:
 301 *      echo 1 > /sys/kernel/debug/<device number>/inject_ecc
 302 */
 303static ssize_t thunderx_lmc_inject_int_write(struct file *file,
 304                                             const char __user *data,
 305                                             size_t count, loff_t *ppos)
 306{
 307        struct thunderx_lmc *lmc = file->private_data;
 308        u64 val;
 309        int res;
 310
 311        res = kstrtoull_from_user(data, count, 0, &val);
 312
 313        if (!res) {
 314                /* Trigger the interrupt */
 315                writeq(val, lmc->regs + LMC_INT_W1S);
 316                res = count;
 317        }
 318
 319        return res;
 320}
 321
 322static ssize_t thunderx_lmc_int_read(struct file *file,
 323                                     char __user *data,
 324                                     size_t count, loff_t *ppos)
 325{
 326        struct thunderx_lmc *lmc = file->private_data;
 327        char buf[20];
 328        u64 lmc_int = readq(lmc->regs + LMC_INT);
 329
 330        snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
 331        return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
 332}
 333
 334#define TEST_PATTERN 0xa5
 335
 336static int inject_ecc_fn(void *arg)
 337{
 338        struct thunderx_lmc *lmc = arg;
 339        uintptr_t addr, phys;
 340        unsigned int cline_size = cache_line_size();
 341        const unsigned int lines = PAGE_SIZE / cline_size;
 342        unsigned int i, cl_idx;
 343
 344        addr = (uintptr_t)page_address(lmc->mem);
 345        phys = (uintptr_t)page_to_phys(lmc->mem);
 346
 347        cl_idx = (phys & 0x7f) >> 4;
 348        lmc->parity_test &= ~(7ULL << 8);
 349        lmc->parity_test |= (cl_idx << 8);
 350
 351        writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
 352        writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
 353        writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
 354
 355        readq(lmc->regs + LMC_CHAR_MASK0);
 356        readq(lmc->regs + LMC_CHAR_MASK2);
 357        readq(lmc->regs + LMC_ECC_PARITY_TEST);
 358
 359        for (i = 0; i < lines; i++) {
 360                memset((void *)addr, TEST_PATTERN, cline_size);
 361                barrier();
 362
 363                /*
 364                 * Flush L1 cachelines to the PoC (L2).
 365                 * This will cause cacheline eviction to the L2.
 366                 */
 367                asm volatile("dc civac, %0\n"
 368                             "dsb sy\n"
 369                             : : "r"(addr + i * cline_size));
 370        }
 371
 372        for (i = 0; i < lines; i++) {
 373                /*
 374                 * Flush L2 cachelines to the DRAM.
 375                 * This will cause cacheline eviction to the DRAM
 376                 * and ECC corruption according to the masks set.
 377                 */
 378                __asm__ volatile("sys #0,c11,C1,#2, %0\n"
 379                                 : : "r"(phys + i * cline_size));
 380        }
 381
 382        for (i = 0; i < lines; i++) {
 383                /*
 384                 * Invalidate L2 cachelines.
 385                 * The subsequent load will cause cacheline fetch
 386                 * from the DRAM and an error interrupt
 387                 */
 388                __asm__ volatile("sys #0,c11,C1,#1, %0"
 389                                 : : "r"(phys + i * cline_size));
 390        }
 391
 392        for (i = 0; i < lines; i++) {
 393                /*
 394                 * Invalidate L1 cachelines.
 395                 * The subsequent load will cause cacheline fetch
 396                 * from the L2 and/or DRAM
 397                 */
 398                asm volatile("dc ivac, %0\n"
 399                             "dsb sy\n"
 400                             : : "r"(addr + i * cline_size));
 401        }
 402
 403        return 0;
 404}
 405
 406static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
 407                                             const char __user *data,
 408                                             size_t count, loff_t *ppos)
 409{
 410        struct thunderx_lmc *lmc = file->private_data;
 411        unsigned int cline_size = cache_line_size();
 412        u8 *tmp;
 413        void __iomem *addr;
 414        unsigned int offs, timeout = 100000;
 415
 416        atomic_set(&lmc->ecc_int, 0);
 417
 418        lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
 419        if (!lmc->mem)
 420                return -ENOMEM;
 421
 422        tmp = kmalloc(cline_size, GFP_KERNEL);
 423        if (!tmp) {
 424                __free_pages(lmc->mem, 0);
 425                return -ENOMEM;
 426        }
 427
 428        addr = page_address(lmc->mem);
 429
 430        while (!atomic_read(&lmc->ecc_int) && timeout--) {
 431                stop_machine(inject_ecc_fn, lmc, NULL);
 432
 433                for (offs = 0; offs < PAGE_SIZE; offs += cline_size) {
 434                        /*
 435                         * Do a load from the previously rigged location
 436                         * This should generate an error interrupt.
 437                         */
 438                        memcpy(tmp, addr + offs, cline_size);
 439                        asm volatile("dsb ld\n");
 440                }
 441        }
 442
 443        kfree(tmp);
 444        __free_pages(lmc->mem, 0);
 445
 446        return count;
 447}
 448
 449LMC_DEBUGFS_ENT(mask0);
 450LMC_DEBUGFS_ENT(mask2);
 451LMC_DEBUGFS_ENT(parity_test);
 452
 453DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
 454DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
 455DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
 456
 457static struct debugfs_entry *lmc_dfs_ents[] = {
 458        &debugfs_mask0,
 459        &debugfs_mask2,
 460        &debugfs_parity_test,
 461        &debugfs_inject_ecc,
 462        &debugfs_inject_int,
 463        &debugfs_int_w1c,
 464};
 465
 466static int thunderx_create_debugfs_nodes(struct dentry *parent,
 467                                          struct debugfs_entry *attrs[],
 468                                          void *data,
 469                                          size_t num)
 470{
 471        int i;
 472        struct dentry *ent;
 473
 474        if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
 475                return 0;
 476
 477        if (!parent)
 478                return -ENOENT;
 479
 480        for (i = 0; i < num; i++) {
 481                ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
 482                                               parent, data, &attrs[i]->fops);
 483
 484                if (!ent)
 485                        break;
 486        }
 487
 488        return i;
 489}
 490
 491static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
 492{
 493        phys_addr_t addr = 0;
 494        int bank, xbits;
 495
 496        addr |= lmc->node << 40;
 497        addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
 498        addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
 499        addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
 500        addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
 501
 502        bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
 503
 504        if (lmc->xor_bank)
 505                bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
 506
 507        addr |= bank << lmc->bank_lsb;
 508
 509        xbits = PCI_FUNC(lmc->pdev->devfn);
 510
 511        if (lmc->l2c_alias)
 512                xbits ^= get_bits(addr, 20, lmc->xbits) ^
 513                         get_bits(addr, 12, lmc->xbits);
 514
 515        addr |= xbits << 7;
 516
 517        return addr;
 518}
 519
 520static unsigned int thunderx_get_num_lmcs(unsigned int node)
 521{
 522        unsigned int number = 0;
 523        struct pci_dev *pdev = NULL;
 524
 525        do {
 526                pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
 527                                      PCI_DEVICE_ID_THUNDER_LMC,
 528                                      pdev);
 529                if (pdev) {
 530#ifdef CONFIG_NUMA
 531                        if (pdev->dev.numa_node == node)
 532                                number++;
 533#else
 534                        number++;
 535#endif
 536                }
 537        } while (pdev);
 538
 539        return number;
 540}
 541
 542#define LMC_MESSAGE_SIZE        120
 543#define LMC_OTHER_SIZE          (50 * ARRAY_SIZE(lmc_errors))
 544
 545static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
 546{
 547        struct mem_ctl_info *mci = dev_id;
 548        struct thunderx_lmc *lmc = mci->pvt_info;
 549
 550        unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
 551        struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
 552
 553        writeq(0, lmc->regs + LMC_CHAR_MASK0);
 554        writeq(0, lmc->regs + LMC_CHAR_MASK2);
 555        writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
 556
 557        ctx->reg_int = readq(lmc->regs + LMC_INT);
 558        ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
 559        ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
 560        ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
 561        ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
 562
 563        lmc->ring_head++;
 564
 565        atomic_set(&lmc->ecc_int, 1);
 566
 567        /* Clear the interrupt */
 568        writeq(ctx->reg_int, lmc->regs + LMC_INT);
 569
 570        return IRQ_WAKE_THREAD;
 571}
 572
 573static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
 574{
 575        struct mem_ctl_info *mci = dev_id;
 576        struct thunderx_lmc *lmc = mci->pvt_info;
 577        phys_addr_t phys_addr;
 578
 579        unsigned long tail;
 580        struct lmc_err_ctx *ctx;
 581
 582        irqreturn_t ret = IRQ_NONE;
 583
 584        char *msg;
 585        char *other;
 586
 587        msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
 588        other =  kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
 589
 590        if (!msg || !other)
 591                goto err_free;
 592
 593        while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
 594                ARRAY_SIZE(lmc->err_ctx))) {
 595                tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
 596
 597                ctx = &lmc->err_ctx[tail];
 598
 599                dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
 600                        ctx->reg_int);
 601                dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
 602                        ctx->reg_fadr);
 603                dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
 604                        ctx->reg_nxm_fadr);
 605                dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
 606                        ctx->reg_scram_fadr);
 607                dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
 608                        ctx->reg_ecc_synd);
 609
 610                snprintf(msg, LMC_MESSAGE_SIZE,
 611                         "DIMM %lld rank %lld bank %lld row %lld col %lld",
 612                         LMC_FADR_FDIMM(ctx->reg_scram_fadr),
 613                         LMC_FADR_FBUNK(ctx->reg_scram_fadr),
 614                         LMC_FADR_FBANK(ctx->reg_scram_fadr),
 615                         LMC_FADR_FROW(ctx->reg_scram_fadr),
 616                         LMC_FADR_FCOL(ctx->reg_scram_fadr));
 617
 618                decode_register(other, LMC_OTHER_SIZE, lmc_errors,
 619                                ctx->reg_int);
 620
 621                phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
 622
 623                if (ctx->reg_int & LMC_INT_UE)
 624                        edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
 625                                             phys_to_pfn(phys_addr),
 626                                             offset_in_page(phys_addr),
 627                                             0, -1, -1, -1, msg, other);
 628                else if (ctx->reg_int & LMC_INT_CE)
 629                        edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
 630                                             phys_to_pfn(phys_addr),
 631                                             offset_in_page(phys_addr),
 632                                             0, -1, -1, -1, msg, other);
 633
 634                lmc->ring_tail++;
 635        }
 636
 637        ret = IRQ_HANDLED;
 638
 639err_free:
 640        kfree(msg);
 641        kfree(other);
 642
 643        return ret;
 644}
 645
 646static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
 647        { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
 648        { 0, },
 649};
 650
 651static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
 652{
 653        int node = dev_to_node(&pdev->dev);
 654        int ret = PCI_FUNC(pdev->devfn);
 655
 656        ret += max(node, 0) << 3;
 657
 658        return ret;
 659}
 660
 661static int thunderx_lmc_probe(struct pci_dev *pdev,
 662                                const struct pci_device_id *id)
 663{
 664        struct thunderx_lmc *lmc;
 665        struct edac_mc_layer layer;
 666        struct mem_ctl_info *mci;
 667        u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
 668        int ret;
 669        u64 lmc_int;
 670        void *l2c_ioaddr;
 671
 672        layer.type = EDAC_MC_LAYER_SLOT;
 673        layer.size = 2;
 674        layer.is_virt_csrow = false;
 675
 676        ret = pcim_enable_device(pdev);
 677        if (ret) {
 678                dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
 679                return ret;
 680        }
 681
 682        ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
 683        if (ret) {
 684                dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
 685                return ret;
 686        }
 687
 688        mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
 689                            sizeof(struct thunderx_lmc));
 690        if (!mci)
 691                return -ENOMEM;
 692
 693        mci->pdev = &pdev->dev;
 694        lmc = mci->pvt_info;
 695
 696        pci_set_drvdata(pdev, mci);
 697
 698        lmc->regs = pcim_iomap_table(pdev)[0];
 699
 700        lmc_control = readq(lmc->regs + LMC_CONTROL);
 701        lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
 702        lmc_config = readq(lmc->regs + LMC_CONFIG);
 703
 704        if (lmc_control & LMC_CONTROL_RDIMM) {
 705                mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
 706                                           lmc_ddr_pll_ctl) ?
 707                                MEM_RDDR4 : MEM_RDDR3;
 708        } else {
 709                mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
 710                                           lmc_ddr_pll_ctl) ?
 711                                MEM_DDR4 : MEM_DDR3;
 712        }
 713
 714        mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 715        mci->edac_cap = EDAC_FLAG_SECDED;
 716
 717        mci->mod_name = "thunderx-lmc";
 718        mci->ctl_name = "thunderx-lmc";
 719        mci->dev_name = dev_name(&pdev->dev);
 720        mci->scrub_mode = SCRUB_NONE;
 721
 722        lmc->pdev = pdev;
 723        lmc->msix_ent.entry = 0;
 724
 725        lmc->ring_head = 0;
 726        lmc->ring_tail = 0;
 727
 728        ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
 729        if (ret) {
 730                dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
 731                goto err_free;
 732        }
 733
 734        ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
 735                                        thunderx_lmc_err_isr,
 736                                        thunderx_lmc_threaded_isr, 0,
 737                                        "[EDAC] ThunderX LMC", mci);
 738        if (ret) {
 739                dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
 740                goto err_free;
 741        }
 742
 743        lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
 744
 745        lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
 746        lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
 747                           FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
 748
 749        lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
 750        lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;
 751        lmc->rank_lsb = lmc->dimm_lsb;
 752        lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
 753        lmc->bank_lsb = 7 + lmc->xbits;
 754        lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
 755
 756        lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
 757
 758        lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
 759
 760        l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
 761        if (!l2c_ioaddr) {
 762                dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
 763                ret = -ENOMEM;
 764                goto err_free;
 765        }
 766
 767        lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
 768
 769        iounmap(l2c_ioaddr);
 770
 771        ret = edac_mc_add_mc(mci);
 772        if (ret) {
 773                dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
 774                goto err_free;
 775        }
 776
 777        lmc_int = readq(lmc->regs + LMC_INT);
 778        writeq(lmc_int, lmc->regs + LMC_INT);
 779
 780        writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
 781
 782        if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
 783                ret = thunderx_create_debugfs_nodes(mci->debugfs,
 784                                                    lmc_dfs_ents,
 785                                                    lmc,
 786                                                    ARRAY_SIZE(lmc_dfs_ents));
 787
 788                if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
 789                        dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
 790                                 ret, ret >= 0 ? " created" : "");
 791                }
 792        }
 793
 794        return 0;
 795
 796err_free:
 797        pci_set_drvdata(pdev, NULL);
 798        edac_mc_free(mci);
 799
 800        return ret;
 801}
 802
 803static void thunderx_lmc_remove(struct pci_dev *pdev)
 804{
 805        struct mem_ctl_info *mci = pci_get_drvdata(pdev);
 806        struct thunderx_lmc *lmc = mci->pvt_info;
 807
 808        writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
 809
 810        edac_mc_del_mc(&pdev->dev);
 811        edac_mc_free(mci);
 812}
 813
 814MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
 815
 816static struct pci_driver thunderx_lmc_driver = {
 817        .name     = "thunderx_lmc_edac",
 818        .probe    = thunderx_lmc_probe,
 819        .remove   = thunderx_lmc_remove,
 820        .id_table = thunderx_lmc_pci_tbl,
 821};
 822
 823/*---------------------- OCX driver ---------------------------------*/
 824
 825#define PCI_DEVICE_ID_THUNDER_OCX 0xa013
 826
 827#define OCX_LINK_INTS           3
 828#define OCX_INTS                (OCX_LINK_INTS + 1)
 829#define OCX_RX_LANES            24
 830#define OCX_RX_LANE_STATS       15
 831
 832#define OCX_COM_INT             0x100
 833#define OCX_COM_INT_W1S         0x108
 834#define OCX_COM_INT_ENA_W1S     0x110
 835#define OCX_COM_INT_ENA_W1C     0x118
 836
 837#define OCX_COM_IO_BADID                BIT(54)
 838#define OCX_COM_MEM_BADID               BIT(53)
 839#define OCX_COM_COPR_BADID              BIT(52)
 840#define OCX_COM_WIN_REQ_BADID           BIT(51)
 841#define OCX_COM_WIN_REQ_TOUT            BIT(50)
 842#define OCX_COM_RX_LANE                 GENMASK(23, 0)
 843
 844#define OCX_COM_INT_CE                  (OCX_COM_IO_BADID      | \
 845                                         OCX_COM_MEM_BADID     | \
 846                                         OCX_COM_COPR_BADID    | \
 847                                         OCX_COM_WIN_REQ_BADID | \
 848                                         OCX_COM_WIN_REQ_TOUT)
 849
 850static const struct error_descr ocx_com_errors[] = {
 851        {
 852                .type  = ERR_CORRECTED,
 853                .mask  = OCX_COM_IO_BADID,
 854                .descr = "Invalid IO transaction node ID",
 855        },
 856        {
 857                .type  = ERR_CORRECTED,
 858                .mask  = OCX_COM_MEM_BADID,
 859                .descr = "Invalid memory transaction node ID",
 860        },
 861        {
 862                .type  = ERR_CORRECTED,
 863                .mask  = OCX_COM_COPR_BADID,
 864                .descr = "Invalid coprocessor transaction node ID",
 865        },
 866        {
 867                .type  = ERR_CORRECTED,
 868                .mask  = OCX_COM_WIN_REQ_BADID,
 869                .descr = "Invalid SLI transaction node ID",
 870        },
 871        {
 872                .type  = ERR_CORRECTED,
 873                .mask  = OCX_COM_WIN_REQ_TOUT,
 874                .descr = "Window/core request timeout",
 875        },
 876        {0, 0, NULL},
 877};
 878
 879#define OCX_COM_LINKX_INT(x)            (0x120 + (x) * 8)
 880#define OCX_COM_LINKX_INT_W1S(x)        (0x140 + (x) * 8)
 881#define OCX_COM_LINKX_INT_ENA_W1S(x)    (0x160 + (x) * 8)
 882#define OCX_COM_LINKX_INT_ENA_W1C(x)    (0x180 + (x) * 8)
 883
 884#define OCX_COM_LINK_BAD_WORD                   BIT(13)
 885#define OCX_COM_LINK_ALIGN_FAIL                 BIT(12)
 886#define OCX_COM_LINK_ALIGN_DONE                 BIT(11)
 887#define OCX_COM_LINK_UP                         BIT(10)
 888#define OCX_COM_LINK_STOP                       BIT(9)
 889#define OCX_COM_LINK_BLK_ERR                    BIT(8)
 890#define OCX_COM_LINK_REINIT                     BIT(7)
 891#define OCX_COM_LINK_LNK_DATA                   BIT(6)
 892#define OCX_COM_LINK_RXFIFO_DBE                 BIT(5)
 893#define OCX_COM_LINK_RXFIFO_SBE                 BIT(4)
 894#define OCX_COM_LINK_TXFIFO_DBE                 BIT(3)
 895#define OCX_COM_LINK_TXFIFO_SBE                 BIT(2)
 896#define OCX_COM_LINK_REPLAY_DBE                 BIT(1)
 897#define OCX_COM_LINK_REPLAY_SBE                 BIT(0)
 898
 899static const struct error_descr ocx_com_link_errors[] = {
 900        {
 901                .type  = ERR_CORRECTED,
 902                .mask  = OCX_COM_LINK_REPLAY_SBE,
 903                .descr = "Replay buffer single-bit error",
 904        },
 905        {
 906                .type  = ERR_CORRECTED,
 907                .mask  = OCX_COM_LINK_TXFIFO_SBE,
 908                .descr = "TX FIFO single-bit error",
 909        },
 910        {
 911                .type  = ERR_CORRECTED,
 912                .mask  = OCX_COM_LINK_RXFIFO_SBE,
 913                .descr = "RX FIFO single-bit error",
 914        },
 915        {
 916                .type  = ERR_CORRECTED,
 917                .mask  = OCX_COM_LINK_BLK_ERR,
 918                .descr = "Block code error",
 919        },
 920        {
 921                .type  = ERR_CORRECTED,
 922                .mask  = OCX_COM_LINK_ALIGN_FAIL,
 923                .descr = "Link alignment failure",
 924        },
 925        {
 926                .type  = ERR_CORRECTED,
 927                .mask  = OCX_COM_LINK_BAD_WORD,
 928                .descr = "Bad code word",
 929        },
 930        {
 931                .type  = ERR_UNCORRECTED,
 932                .mask  = OCX_COM_LINK_REPLAY_DBE,
 933                .descr = "Replay buffer double-bit error",
 934        },
 935        {
 936                .type  = ERR_UNCORRECTED,
 937                .mask  = OCX_COM_LINK_TXFIFO_DBE,
 938                .descr = "TX FIFO double-bit error",
 939        },
 940        {
 941                .type  = ERR_UNCORRECTED,
 942                .mask  = OCX_COM_LINK_RXFIFO_DBE,
 943                .descr = "RX FIFO double-bit error",
 944        },
 945        {
 946                .type  = ERR_UNCORRECTED,
 947                .mask  = OCX_COM_LINK_STOP,
 948                .descr = "Link stopped",
 949        },
 950        {0, 0, NULL},
 951};
 952
 953#define OCX_COM_LINK_INT_UE       (OCX_COM_LINK_REPLAY_DBE | \
 954                                   OCX_COM_LINK_TXFIFO_DBE | \
 955                                   OCX_COM_LINK_RXFIFO_DBE | \
 956                                   OCX_COM_LINK_STOP)
 957
 958#define OCX_COM_LINK_INT_CE       (OCX_COM_LINK_REPLAY_SBE | \
 959                                   OCX_COM_LINK_TXFIFO_SBE | \
 960                                   OCX_COM_LINK_RXFIFO_SBE | \
 961                                   OCX_COM_LINK_BLK_ERR    | \
 962                                   OCX_COM_LINK_ALIGN_FAIL | \
 963                                   OCX_COM_LINK_BAD_WORD)
 964
 965#define OCX_LNE_INT(x)                  (0x8018 + (x) * 0x100)
 966#define OCX_LNE_INT_EN(x)               (0x8020 + (x) * 0x100)
 967#define OCX_LNE_BAD_CNT(x)              (0x8028 + (x) * 0x100)
 968#define OCX_LNE_CFG(x)                  (0x8000 + (x) * 0x100)
 969#define OCX_LNE_STAT(x, y)              (0x8040 + (x) * 0x100 + (y) * 8)
 970
 971#define OCX_LNE_CFG_RX_BDRY_LOCK_DIS            BIT(8)
 972#define OCX_LNE_CFG_RX_STAT_WRAP_DIS            BIT(2)
 973#define OCX_LNE_CFG_RX_STAT_RDCLR               BIT(1)
 974#define OCX_LNE_CFG_RX_STAT_ENA                 BIT(0)
 975
 976
 977#define OCX_LANE_BAD_64B67B                     BIT(8)
 978#define OCX_LANE_DSKEW_FIFO_OVFL                BIT(5)
 979#define OCX_LANE_SCRM_SYNC_LOSS                 BIT(4)
 980#define OCX_LANE_UKWN_CNTL_WORD                 BIT(3)
 981#define OCX_LANE_CRC32_ERR                      BIT(2)
 982#define OCX_LANE_BDRY_SYNC_LOSS                 BIT(1)
 983#define OCX_LANE_SERDES_LOCK_LOSS               BIT(0)
 984
 985#define OCX_COM_LANE_INT_UE       (0)
 986#define OCX_COM_LANE_INT_CE       (OCX_LANE_SERDES_LOCK_LOSS | \
 987                                   OCX_LANE_BDRY_SYNC_LOSS   | \
 988                                   OCX_LANE_CRC32_ERR        | \
 989                                   OCX_LANE_UKWN_CNTL_WORD   | \
 990                                   OCX_LANE_SCRM_SYNC_LOSS   | \
 991                                   OCX_LANE_DSKEW_FIFO_OVFL  | \
 992                                   OCX_LANE_BAD_64B67B)
 993
 994static const struct error_descr ocx_lane_errors[] = {
 995        {
 996                .type  = ERR_CORRECTED,
 997                .mask  = OCX_LANE_SERDES_LOCK_LOSS,
 998                .descr = "RX SerDes lock lost",
 999        },
1000        {
1001                .type  = ERR_CORRECTED,
1002                .mask  = OCX_LANE_BDRY_SYNC_LOSS,
1003                .descr = "RX word boundary lost",
1004        },
1005        {
1006                .type  = ERR_CORRECTED,
1007                .mask  = OCX_LANE_CRC32_ERR,
1008                .descr = "CRC32 error",
1009        },
1010        {
1011                .type  = ERR_CORRECTED,
1012                .mask  = OCX_LANE_UKWN_CNTL_WORD,
1013                .descr = "Unknown control word",
1014        },
1015        {
1016                .type  = ERR_CORRECTED,
1017                .mask  = OCX_LANE_SCRM_SYNC_LOSS,
1018                .descr = "Scrambler synchronization lost",
1019        },
1020        {
1021                .type  = ERR_CORRECTED,
1022                .mask  = OCX_LANE_DSKEW_FIFO_OVFL,
1023                .descr = "RX deskew FIFO overflow",
1024        },
1025        {
1026                .type  = ERR_CORRECTED,
1027                .mask  = OCX_LANE_BAD_64B67B,
1028                .descr = "Bad 64B/67B codeword",
1029        },
1030        {0, 0, NULL},
1031};
1032
1033#define OCX_LNE_INT_ENA_ALL             (GENMASK(9, 8) | GENMASK(6, 0))
1034#define OCX_COM_INT_ENA_ALL             (GENMASK(54, 50) | GENMASK(23, 0))
1035#define OCX_COM_LINKX_INT_ENA_ALL       (GENMASK(13, 12) | \
1036                                         GENMASK(9, 7) | GENMASK(5, 0))
1037
1038#define OCX_TLKX_ECC_CTL(x)             (0x10018 + (x) * 0x2000)
1039#define OCX_RLKX_ECC_CTL(x)             (0x18018 + (x) * 0x2000)
1040
1041struct ocx_com_err_ctx {
1042        u64 reg_com_int;
1043        u64 reg_lane_int[OCX_RX_LANES];
1044        u64 reg_lane_stat11[OCX_RX_LANES];
1045};
1046
1047struct ocx_link_err_ctx {
1048        u64 reg_com_link_int;
1049        int link;
1050};
1051
1052struct thunderx_ocx {
1053        void __iomem *regs;
1054        int com_link;
1055        struct pci_dev *pdev;
1056        struct edac_device_ctl_info *edac_dev;
1057
1058        struct dentry *debugfs;
1059        struct msix_entry msix_ent[OCX_INTS];
1060
1061        struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1062        struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1063
1064        unsigned long com_ring_head;
1065        unsigned long com_ring_tail;
1066
1067        unsigned long link_ring_head;
1068        unsigned long link_ring_tail;
1069};
1070
1071#define OCX_MESSAGE_SIZE        SZ_1K
1072#define OCX_OTHER_SIZE          (50 * ARRAY_SIZE(ocx_com_link_errors))
1073
1074/* This handler is threaded */
1075static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1076{
1077        struct msix_entry *msix = irq_id;
1078        struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1079                                                msix_ent[msix->entry]);
1080
1081        int lane;
1082        unsigned long head = ring_pos(ocx->com_ring_head,
1083                                      ARRAY_SIZE(ocx->com_err_ctx));
1084        struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1085
1086        ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1087
1088        for (lane = 0; lane < OCX_RX_LANES; lane++) {
1089                ctx->reg_lane_int[lane] =
1090                        readq(ocx->regs + OCX_LNE_INT(lane));
1091                ctx->reg_lane_stat11[lane] =
1092                        readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1093
1094                writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1095        }
1096
1097        writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1098
1099        ocx->com_ring_head++;
1100
1101        return IRQ_WAKE_THREAD;
1102}
1103
1104static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1105{
1106        struct msix_entry *msix = irq_id;
1107        struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1108                                                msix_ent[msix->entry]);
1109
1110        irqreturn_t ret = IRQ_NONE;
1111
1112        unsigned long tail;
1113        struct ocx_com_err_ctx *ctx;
1114        int lane;
1115        char *msg;
1116        char *other;
1117
1118        msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1119        other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1120
1121        if (!msg || !other)
1122                goto err_free;
1123
1124        while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1125                        ARRAY_SIZE(ocx->com_err_ctx))) {
1126                tail = ring_pos(ocx->com_ring_tail,
1127                                ARRAY_SIZE(ocx->com_err_ctx));
1128                ctx = &ocx->com_err_ctx[tail];
1129
1130                snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1131                        ocx->edac_dev->ctl_name, ctx->reg_com_int);
1132
1133                decode_register(other, OCX_OTHER_SIZE,
1134                                ocx_com_errors, ctx->reg_com_int);
1135
1136                strncat(msg, other, OCX_MESSAGE_SIZE);
1137
1138                for (lane = 0; lane < OCX_RX_LANES; lane++)
1139                        if (ctx->reg_com_int & BIT(lane)) {
1140                                snprintf(other, OCX_OTHER_SIZE,
1141                                         "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1142                                         lane, ctx->reg_lane_int[lane],
1143                                         lane, ctx->reg_lane_stat11[lane]);
1144
1145                                strncat(msg, other, OCX_MESSAGE_SIZE);
1146
1147                                decode_register(other, OCX_OTHER_SIZE,
1148                                                ocx_lane_errors,
1149                                                ctx->reg_lane_int[lane]);
1150                                strncat(msg, other, OCX_MESSAGE_SIZE);
1151                        }
1152
1153                if (ctx->reg_com_int & OCX_COM_INT_CE)
1154                        edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1155
1156                ocx->com_ring_tail++;
1157        }
1158
1159        ret = IRQ_HANDLED;
1160
1161err_free:
1162        kfree(other);
1163        kfree(msg);
1164
1165        return ret;
1166}
1167
1168static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1169{
1170        struct msix_entry *msix = irq_id;
1171        struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1172                                                msix_ent[msix->entry]);
1173        unsigned long head = ring_pos(ocx->link_ring_head,
1174                                      ARRAY_SIZE(ocx->link_err_ctx));
1175        struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1176
1177        ctx->link = msix->entry;
1178        ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1179
1180        writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1181
1182        ocx->link_ring_head++;
1183
1184        return IRQ_WAKE_THREAD;
1185}
1186
1187static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1188{
1189        struct msix_entry *msix = irq_id;
1190        struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1191                                                msix_ent[msix->entry]);
1192        irqreturn_t ret = IRQ_NONE;
1193        unsigned long tail;
1194        struct ocx_link_err_ctx *ctx;
1195
1196        char *msg;
1197        char *other;
1198
1199        msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1200        other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1201
1202        if (!msg || !other)
1203                goto err_free;
1204
1205        while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1206                        ARRAY_SIZE(ocx->link_err_ctx))) {
1207                tail = ring_pos(ocx->link_ring_head,
1208                                ARRAY_SIZE(ocx->link_err_ctx));
1209
1210                ctx = &ocx->link_err_ctx[tail];
1211
1212                snprintf(msg, OCX_MESSAGE_SIZE,
1213                         "%s: OCX_COM_LINK_INT[%d]: %016llx",
1214                         ocx->edac_dev->ctl_name,
1215                         ctx->link, ctx->reg_com_link_int);
1216
1217                decode_register(other, OCX_OTHER_SIZE,
1218                                ocx_com_link_errors, ctx->reg_com_link_int);
1219
1220                strncat(msg, other, OCX_MESSAGE_SIZE);
1221
1222                if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1223                        edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1224                else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1225                        edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1226
1227                ocx->link_ring_tail++;
1228        }
1229
1230        ret = IRQ_HANDLED;
1231err_free:
1232        kfree(other);
1233        kfree(msg);
1234
1235        return ret;
1236}
1237
1238#define OCX_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(ocx, _name, _reg)
1239
1240OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1241OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1242OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1243
1244OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1245OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1246OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1247
1248OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1249OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1250OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1251
1252OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1253OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1254OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1255OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1256OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1257OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1258OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1259OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1260
1261OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1262OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1263OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1264OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1265OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1266OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1267OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1268OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1269
1270OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1271OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1272OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1273OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1274OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1275OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1276OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1277OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1278
1279OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1280
1281static struct debugfs_entry *ocx_dfs_ents[] = {
1282        &debugfs_tlk0_ecc_ctl,
1283        &debugfs_tlk1_ecc_ctl,
1284        &debugfs_tlk2_ecc_ctl,
1285
1286        &debugfs_rlk0_ecc_ctl,
1287        &debugfs_rlk1_ecc_ctl,
1288        &debugfs_rlk2_ecc_ctl,
1289
1290        &debugfs_com_link0_int,
1291        &debugfs_com_link1_int,
1292        &debugfs_com_link2_int,
1293
1294        &debugfs_lne00_badcnt,
1295        &debugfs_lne01_badcnt,
1296        &debugfs_lne02_badcnt,
1297        &debugfs_lne03_badcnt,
1298        &debugfs_lne04_badcnt,
1299        &debugfs_lne05_badcnt,
1300        &debugfs_lne06_badcnt,
1301        &debugfs_lne07_badcnt,
1302        &debugfs_lne08_badcnt,
1303        &debugfs_lne09_badcnt,
1304        &debugfs_lne10_badcnt,
1305        &debugfs_lne11_badcnt,
1306        &debugfs_lne12_badcnt,
1307        &debugfs_lne13_badcnt,
1308        &debugfs_lne14_badcnt,
1309        &debugfs_lne15_badcnt,
1310        &debugfs_lne16_badcnt,
1311        &debugfs_lne17_badcnt,
1312        &debugfs_lne18_badcnt,
1313        &debugfs_lne19_badcnt,
1314        &debugfs_lne20_badcnt,
1315        &debugfs_lne21_badcnt,
1316        &debugfs_lne22_badcnt,
1317        &debugfs_lne23_badcnt,
1318
1319        &debugfs_com_int,
1320};
1321
1322static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1323        { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1324        { 0, },
1325};
1326
1327static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1328{
1329        int lane, stat, cfg;
1330
1331        for (lane = 0; lane < OCX_RX_LANES; lane++) {
1332                cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1333                cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1334                cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1335                writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1336
1337                for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1338                        readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1339        }
1340}
1341
1342static int thunderx_ocx_probe(struct pci_dev *pdev,
1343                              const struct pci_device_id *id)
1344{
1345        struct thunderx_ocx *ocx;
1346        struct edac_device_ctl_info *edac_dev;
1347        char name[32];
1348        int idx;
1349        int i;
1350        int ret;
1351        u64 reg;
1352
1353        ret = pcim_enable_device(pdev);
1354        if (ret) {
1355                dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1356                return ret;
1357        }
1358
1359        ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1360        if (ret) {
1361                dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1362                return ret;
1363        }
1364
1365        idx = edac_device_alloc_index();
1366        snprintf(name, sizeof(name), "OCX%d", idx);
1367        edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1368                                              name, 1, "CCPI", 1,
1369                                              0, NULL, 0, idx);
1370        if (!edac_dev) {
1371                dev_err(&pdev->dev, "Cannot allocate EDAC device: %d\n", ret);
1372                return -ENOMEM;
1373        }
1374        ocx = edac_dev->pvt_info;
1375        ocx->edac_dev = edac_dev;
1376        ocx->com_ring_head = 0;
1377        ocx->com_ring_tail = 0;
1378        ocx->link_ring_head = 0;
1379        ocx->link_ring_tail = 0;
1380
1381        ocx->regs = pcim_iomap_table(pdev)[0];
1382        if (!ocx->regs) {
1383                dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1384                ret = -ENODEV;
1385                goto err_free;
1386        }
1387
1388        ocx->pdev = pdev;
1389
1390        for (i = 0; i < OCX_INTS; i++) {
1391                ocx->msix_ent[i].entry = i;
1392                ocx->msix_ent[i].vector = 0;
1393        }
1394
1395        ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1396        if (ret) {
1397                dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1398                goto err_free;
1399        }
1400
1401        for (i = 0; i < OCX_INTS; i++) {
1402                ret = devm_request_threaded_irq(&pdev->dev,
1403                                                ocx->msix_ent[i].vector,
1404                                                (i == 3) ?
1405                                                 thunderx_ocx_com_isr :
1406                                                 thunderx_ocx_lnk_isr,
1407                                                (i == 3) ?
1408                                                 thunderx_ocx_com_threaded_isr :
1409                                                 thunderx_ocx_lnk_threaded_isr,
1410                                                0, "[EDAC] ThunderX OCX",
1411                                                &ocx->msix_ent[i]);
1412                if (ret)
1413                        goto err_free;
1414        }
1415
1416        edac_dev->dev = &pdev->dev;
1417        edac_dev->dev_name = dev_name(&pdev->dev);
1418        edac_dev->mod_name = "thunderx-ocx";
1419        edac_dev->ctl_name = "thunderx-ocx";
1420
1421        ret = edac_device_add_device(edac_dev);
1422        if (ret) {
1423                dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1424                goto err_free;
1425        }
1426
1427        if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1428                ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1429
1430                ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1431                                                    ocx_dfs_ents,
1432                                                    ocx,
1433                                                    ARRAY_SIZE(ocx_dfs_ents));
1434                if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1435                        dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1436                                 ret, ret >= 0 ? " created" : "");
1437                }
1438        }
1439
1440        pci_set_drvdata(pdev, edac_dev);
1441
1442        thunderx_ocx_clearstats(ocx);
1443
1444        for (i = 0; i < OCX_RX_LANES; i++) {
1445                writeq(OCX_LNE_INT_ENA_ALL,
1446                       ocx->regs + OCX_LNE_INT_EN(i));
1447
1448                reg = readq(ocx->regs + OCX_LNE_INT(i));
1449                writeq(reg, ocx->regs + OCX_LNE_INT(i));
1450
1451        }
1452
1453        for (i = 0; i < OCX_LINK_INTS; i++) {
1454                reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1455                writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1456
1457                writeq(OCX_COM_LINKX_INT_ENA_ALL,
1458                       ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1459        }
1460
1461        reg = readq(ocx->regs + OCX_COM_INT);
1462        writeq(reg, ocx->regs + OCX_COM_INT);
1463
1464        writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1465
1466        return 0;
1467err_free:
1468        edac_device_free_ctl_info(edac_dev);
1469
1470        return ret;
1471}
1472
1473static void thunderx_ocx_remove(struct pci_dev *pdev)
1474{
1475        struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1476        struct thunderx_ocx *ocx = edac_dev->pvt_info;
1477        int i;
1478
1479        writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1480
1481        for (i = 0; i < OCX_INTS; i++) {
1482                writeq(OCX_COM_LINKX_INT_ENA_ALL,
1483                       ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1484        }
1485
1486        edac_debugfs_remove_recursive(ocx->debugfs);
1487
1488        edac_device_del_device(&pdev->dev);
1489        edac_device_free_ctl_info(edac_dev);
1490}
1491
1492MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1493
1494static struct pci_driver thunderx_ocx_driver = {
1495        .name     = "thunderx_ocx_edac",
1496        .probe    = thunderx_ocx_probe,
1497        .remove   = thunderx_ocx_remove,
1498        .id_table = thunderx_ocx_pci_tbl,
1499};
1500
1501/*---------------------- L2C driver ---------------------------------*/
1502
1503#define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1504#define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1505#define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1506
1507#define L2C_TAD_INT_W1C         0x40000
1508#define L2C_TAD_INT_W1S         0x40008
1509
1510#define L2C_TAD_INT_ENA_W1C     0x40020
1511#define L2C_TAD_INT_ENA_W1S     0x40028
1512
1513
1514#define L2C_TAD_INT_L2DDBE       BIT(1)
1515#define L2C_TAD_INT_SBFSBE       BIT(2)
1516#define L2C_TAD_INT_SBFDBE       BIT(3)
1517#define L2C_TAD_INT_FBFSBE       BIT(4)
1518#define L2C_TAD_INT_FBFDBE       BIT(5)
1519#define L2C_TAD_INT_TAGDBE       BIT(9)
1520#define L2C_TAD_INT_RDDISLMC     BIT(15)
1521#define L2C_TAD_INT_WRDISLMC     BIT(16)
1522#define L2C_TAD_INT_LFBTO        BIT(17)
1523#define L2C_TAD_INT_GSYNCTO      BIT(18)
1524#define L2C_TAD_INT_RTGSBE       BIT(32)
1525#define L2C_TAD_INT_RTGDBE       BIT(33)
1526#define L2C_TAD_INT_RDDISOCI     BIT(34)
1527#define L2C_TAD_INT_WRDISOCI     BIT(35)
1528
1529#define L2C_TAD_INT_ECC         (L2C_TAD_INT_L2DDBE | \
1530                                 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1531                                 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1532
1533#define L2C_TAD_INT_CE          (L2C_TAD_INT_SBFSBE | \
1534                                 L2C_TAD_INT_FBFSBE)
1535
1536#define L2C_TAD_INT_UE          (L2C_TAD_INT_L2DDBE | \
1537                                 L2C_TAD_INT_SBFDBE | \
1538                                 L2C_TAD_INT_FBFDBE | \
1539                                 L2C_TAD_INT_TAGDBE | \
1540                                 L2C_TAD_INT_RTGDBE | \
1541                                 L2C_TAD_INT_WRDISOCI | \
1542                                 L2C_TAD_INT_RDDISOCI | \
1543                                 L2C_TAD_INT_WRDISLMC | \
1544                                 L2C_TAD_INT_RDDISLMC | \
1545                                 L2C_TAD_INT_LFBTO    | \
1546                                 L2C_TAD_INT_GSYNCTO)
1547
1548static const struct error_descr l2_tad_errors[] = {
1549        {
1550                .type  = ERR_CORRECTED,
1551                .mask  = L2C_TAD_INT_SBFSBE,
1552                .descr = "SBF single-bit error",
1553        },
1554        {
1555                .type  = ERR_CORRECTED,
1556                .mask  = L2C_TAD_INT_FBFSBE,
1557                .descr = "FBF single-bit error",
1558        },
1559        {
1560                .type  = ERR_UNCORRECTED,
1561                .mask  = L2C_TAD_INT_L2DDBE,
1562                .descr = "L2D double-bit error",
1563        },
1564        {
1565                .type  = ERR_UNCORRECTED,
1566                .mask  = L2C_TAD_INT_SBFDBE,
1567                .descr = "SBF double-bit error",
1568        },
1569        {
1570                .type  = ERR_UNCORRECTED,
1571                .mask  = L2C_TAD_INT_FBFDBE,
1572                .descr = "FBF double-bit error",
1573        },
1574        {
1575                .type  = ERR_UNCORRECTED,
1576                .mask  = L2C_TAD_INT_TAGDBE,
1577                .descr = "TAG double-bit error",
1578        },
1579        {
1580                .type  = ERR_UNCORRECTED,
1581                .mask  = L2C_TAD_INT_RTGDBE,
1582                .descr = "RTG double-bit error",
1583        },
1584        {
1585                .type  = ERR_UNCORRECTED,
1586                .mask  = L2C_TAD_INT_WRDISOCI,
1587                .descr = "Write to a disabled CCPI",
1588        },
1589        {
1590                .type  = ERR_UNCORRECTED,
1591                .mask  = L2C_TAD_INT_RDDISOCI,
1592                .descr = "Read from a disabled CCPI",
1593        },
1594        {
1595                .type  = ERR_UNCORRECTED,
1596                .mask  = L2C_TAD_INT_WRDISLMC,
1597                .descr = "Write to a disabled LMC",
1598        },
1599        {
1600                .type  = ERR_UNCORRECTED,
1601                .mask  = L2C_TAD_INT_RDDISLMC,
1602                .descr = "Read from a disabled LMC",
1603        },
1604        {
1605                .type  = ERR_UNCORRECTED,
1606                .mask  = L2C_TAD_INT_LFBTO,
1607                .descr = "LFB entry timeout",
1608        },
1609        {
1610                .type  = ERR_UNCORRECTED,
1611                .mask  = L2C_TAD_INT_GSYNCTO,
1612                .descr = "Global sync CCPI timeout",
1613        },
1614        {0, 0, NULL},
1615};
1616
1617#define L2C_TAD_INT_TAG         (L2C_TAD_INT_TAGDBE)
1618
1619#define L2C_TAD_INT_RTG         (L2C_TAD_INT_RTGDBE)
1620
1621#define L2C_TAD_INT_DISLMC      (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1622
1623#define L2C_TAD_INT_DISOCI      (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1624
1625#define L2C_TAD_INT_ENA_ALL     (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1626                                 L2C_TAD_INT_RTG | \
1627                                 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1628                                 L2C_TAD_INT_LFBTO)
1629
1630#define L2C_TAD_TIMETWO         0x50000
1631#define L2C_TAD_TIMEOUT         0x50100
1632#define L2C_TAD_ERR             0x60000
1633#define L2C_TAD_TQD_ERR         0x60100
1634#define L2C_TAD_TTG_ERR         0x60200
1635
1636
1637#define L2C_CBC_INT_W1C         0x60000
1638
1639#define L2C_CBC_INT_RSDSBE       BIT(0)
1640#define L2C_CBC_INT_RSDDBE       BIT(1)
1641
1642#define L2C_CBC_INT_RSD          (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1643
1644#define L2C_CBC_INT_MIBSBE       BIT(4)
1645#define L2C_CBC_INT_MIBDBE       BIT(5)
1646
1647#define L2C_CBC_INT_MIB          (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1648
1649#define L2C_CBC_INT_IORDDISOCI   BIT(6)
1650#define L2C_CBC_INT_IOWRDISOCI   BIT(7)
1651
1652#define L2C_CBC_INT_IODISOCI     (L2C_CBC_INT_IORDDISOCI | \
1653                                  L2C_CBC_INT_IOWRDISOCI)
1654
1655#define L2C_CBC_INT_CE           (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1656#define L2C_CBC_INT_UE           (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1657
1658
1659static const struct error_descr l2_cbc_errors[] = {
1660        {
1661                .type  = ERR_CORRECTED,
1662                .mask  = L2C_CBC_INT_RSDSBE,
1663                .descr = "RSD single-bit error",
1664        },
1665        {
1666                .type  = ERR_CORRECTED,
1667                .mask  = L2C_CBC_INT_MIBSBE,
1668                .descr = "MIB single-bit error",
1669        },
1670        {
1671                .type  = ERR_UNCORRECTED,
1672                .mask  = L2C_CBC_INT_RSDDBE,
1673                .descr = "RSD double-bit error",
1674        },
1675        {
1676                .type  = ERR_UNCORRECTED,
1677                .mask  = L2C_CBC_INT_MIBDBE,
1678                .descr = "MIB double-bit error",
1679        },
1680        {
1681                .type  = ERR_UNCORRECTED,
1682                .mask  = L2C_CBC_INT_IORDDISOCI,
1683                .descr = "Read from a disabled CCPI",
1684        },
1685        {
1686                .type  = ERR_UNCORRECTED,
1687                .mask  = L2C_CBC_INT_IOWRDISOCI,
1688                .descr = "Write to a disabled CCPI",
1689        },
1690        {0, 0, NULL},
1691};
1692
1693#define L2C_CBC_INT_W1S         0x60008
1694#define L2C_CBC_INT_ENA_W1C     0x60020
1695
1696#define L2C_CBC_INT_ENA_ALL      (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1697                                  L2C_CBC_INT_IODISOCI)
1698
1699#define L2C_CBC_INT_ENA_W1S     0x60028
1700
1701#define L2C_CBC_IODISOCIERR     0x80008
1702#define L2C_CBC_IOCERR          0x80010
1703#define L2C_CBC_RSDERR          0x80018
1704#define L2C_CBC_MIBERR          0x80020
1705
1706
1707#define L2C_MCI_INT_W1C         0x0
1708
1709#define L2C_MCI_INT_VBFSBE       BIT(0)
1710#define L2C_MCI_INT_VBFDBE       BIT(1)
1711
1712static const struct error_descr l2_mci_errors[] = {
1713        {
1714                .type  = ERR_CORRECTED,
1715                .mask  = L2C_MCI_INT_VBFSBE,
1716                .descr = "VBF single-bit error",
1717        },
1718        {
1719                .type  = ERR_UNCORRECTED,
1720                .mask  = L2C_MCI_INT_VBFDBE,
1721                .descr = "VBF double-bit error",
1722        },
1723        {0, 0, NULL},
1724};
1725
1726#define L2C_MCI_INT_W1S         0x8
1727#define L2C_MCI_INT_ENA_W1C     0x20
1728
1729#define L2C_MCI_INT_ENA_ALL      (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1730
1731#define L2C_MCI_INT_ENA_W1S     0x28
1732
1733#define L2C_MCI_ERR             0x10000
1734
1735#define L2C_MESSAGE_SIZE        SZ_1K
1736#define L2C_OTHER_SIZE          (50 * ARRAY_SIZE(l2_tad_errors))
1737
1738struct l2c_err_ctx {
1739        char *reg_ext_name;
1740        u64  reg_int;
1741        u64  reg_ext;
1742};
1743
1744struct thunderx_l2c {
1745        void __iomem *regs;
1746        struct pci_dev *pdev;
1747        struct edac_device_ctl_info *edac_dev;
1748
1749        struct dentry *debugfs;
1750
1751        int index;
1752
1753        struct msix_entry msix_ent;
1754
1755        struct l2c_err_ctx err_ctx[RING_ENTRIES];
1756        unsigned long ring_head;
1757        unsigned long ring_tail;
1758};
1759
1760static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1761{
1762        struct msix_entry *msix = irq_id;
1763        struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1764                                                msix_ent);
1765
1766        unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1767        struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1768
1769        ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1770
1771        if (ctx->reg_int & L2C_TAD_INT_ECC) {
1772                ctx->reg_ext_name = "TQD_ERR";
1773                ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1774        } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1775                ctx->reg_ext_name = "TTG_ERR";
1776                ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1777        } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1778                ctx->reg_ext_name = "TIMEOUT";
1779                ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1780        } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1781                ctx->reg_ext_name = "ERR";
1782                ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1783        }
1784
1785        writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1786
1787        tad->ring_head++;
1788
1789        return IRQ_WAKE_THREAD;
1790}
1791
1792static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1793{
1794        struct msix_entry *msix = irq_id;
1795        struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1796                                                msix_ent);
1797
1798        unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1799        struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1800
1801        ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1802
1803        if (ctx->reg_int & L2C_CBC_INT_RSD) {
1804                ctx->reg_ext_name = "RSDERR";
1805                ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1806        } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1807                ctx->reg_ext_name = "MIBERR";
1808                ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1809        } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1810                ctx->reg_ext_name = "IODISOCIERR";
1811                ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1812        }
1813
1814        writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1815
1816        cbc->ring_head++;
1817
1818        return IRQ_WAKE_THREAD;
1819}
1820
1821static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1822{
1823        struct msix_entry *msix = irq_id;
1824        struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1825                                                msix_ent);
1826
1827        unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1828        struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1829
1830        ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1831        ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1832
1833        writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1834
1835        ctx->reg_ext_name = "ERR";
1836
1837        mci->ring_head++;
1838
1839        return IRQ_WAKE_THREAD;
1840}
1841
1842static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1843{
1844        struct msix_entry *msix = irq_id;
1845        struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1846                                                msix_ent);
1847
1848        unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1849        struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1850        irqreturn_t ret = IRQ_NONE;
1851
1852        u64 mask_ue, mask_ce;
1853        const struct error_descr *l2_errors;
1854        char *reg_int_name;
1855
1856        char *msg;
1857        char *other;
1858
1859        msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1860        other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1861
1862        if (!msg || !other)
1863                goto err_free;
1864
1865        switch (l2c->pdev->device) {
1866        case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1867                reg_int_name = "L2C_TAD_INT";
1868                mask_ue = L2C_TAD_INT_UE;
1869                mask_ce = L2C_TAD_INT_CE;
1870                l2_errors = l2_tad_errors;
1871                break;
1872        case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1873                reg_int_name = "L2C_CBC_INT";
1874                mask_ue = L2C_CBC_INT_UE;
1875                mask_ce = L2C_CBC_INT_CE;
1876                l2_errors = l2_cbc_errors;
1877                break;
1878        case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1879                reg_int_name = "L2C_MCI_INT";
1880                mask_ue = L2C_MCI_INT_VBFDBE;
1881                mask_ce = L2C_MCI_INT_VBFSBE;
1882                l2_errors = l2_mci_errors;
1883                break;
1884        default:
1885                dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1886                        l2c->pdev->device);
1887                goto err_free;
1888        }
1889
1890        while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1891                        ARRAY_SIZE(l2c->err_ctx))) {
1892                snprintf(msg, L2C_MESSAGE_SIZE,
1893                         "%s: %s: %016llx, %s: %016llx",
1894                         l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1895                         ctx->reg_ext_name, ctx->reg_ext);
1896
1897                decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1898
1899                strncat(msg, other, L2C_MESSAGE_SIZE);
1900
1901                if (ctx->reg_int & mask_ue)
1902                        edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1903                else if (ctx->reg_int & mask_ce)
1904                        edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1905
1906                l2c->ring_tail++;
1907        }
1908
1909        ret = IRQ_HANDLED;
1910
1911err_free:
1912        kfree(other);
1913        kfree(msg);
1914
1915        return ret;
1916}
1917
1918#define L2C_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(l2c, _name, _reg)
1919
1920L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1921
1922static struct debugfs_entry *l2c_tad_dfs_ents[] = {
1923        &debugfs_tad_int,
1924};
1925
1926L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1927
1928static struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1929        &debugfs_cbc_int,
1930};
1931
1932L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1933
1934static struct debugfs_entry *l2c_mci_dfs_ents[] = {
1935        &debugfs_mci_int,
1936};
1937
1938static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1939        { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1940        { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1941        { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1942        { 0, },
1943};
1944
1945static int thunderx_l2c_probe(struct pci_dev *pdev,
1946                              const struct pci_device_id *id)
1947{
1948        struct thunderx_l2c *l2c;
1949        struct edac_device_ctl_info *edac_dev;
1950        struct debugfs_entry **l2c_devattr;
1951        size_t dfs_entries;
1952        irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1953        char name[32];
1954        const char *fmt;
1955        u64 reg_en_offs, reg_en_mask;
1956        int idx;
1957        int ret;
1958
1959        ret = pcim_enable_device(pdev);
1960        if (ret) {
1961                dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1962                return ret;
1963        }
1964
1965        ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1966        if (ret) {
1967                dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1968                return ret;
1969        }
1970
1971        switch (pdev->device) {
1972        case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1973                thunderx_l2c_isr = thunderx_l2c_tad_isr;
1974                l2c_devattr = l2c_tad_dfs_ents;
1975                dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1976                fmt = "L2C-TAD%d";
1977                reg_en_offs = L2C_TAD_INT_ENA_W1S;
1978                reg_en_mask = L2C_TAD_INT_ENA_ALL;
1979                break;
1980        case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1981                thunderx_l2c_isr = thunderx_l2c_cbc_isr;
1982                l2c_devattr = l2c_cbc_dfs_ents;
1983                dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
1984                fmt = "L2C-CBC%d";
1985                reg_en_offs = L2C_CBC_INT_ENA_W1S;
1986                reg_en_mask = L2C_CBC_INT_ENA_ALL;
1987                break;
1988        case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1989                thunderx_l2c_isr = thunderx_l2c_mci_isr;
1990                l2c_devattr = l2c_mci_dfs_ents;
1991                dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
1992                fmt = "L2C-MCI%d";
1993                reg_en_offs = L2C_MCI_INT_ENA_W1S;
1994                reg_en_mask = L2C_MCI_INT_ENA_ALL;
1995                break;
1996        default:
1997                //Should never ever get here
1998                dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
1999                        pdev->device);
2000                return -EINVAL;
2001        }
2002
2003        idx = edac_device_alloc_index();
2004        snprintf(name, sizeof(name), fmt, idx);
2005
2006        edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2007                                              name, 1, "L2C", 1, 0,
2008                                              NULL, 0, idx);
2009        if (!edac_dev) {
2010                dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2011                return -ENOMEM;
2012        }
2013
2014        l2c = edac_dev->pvt_info;
2015        l2c->edac_dev = edac_dev;
2016
2017        l2c->regs = pcim_iomap_table(pdev)[0];
2018        if (!l2c->regs) {
2019                dev_err(&pdev->dev, "Cannot map PCI resources\n");
2020                ret = -ENODEV;
2021                goto err_free;
2022        }
2023
2024        l2c->pdev = pdev;
2025
2026        l2c->ring_head = 0;
2027        l2c->ring_tail = 0;
2028
2029        l2c->msix_ent.entry = 0;
2030        l2c->msix_ent.vector = 0;
2031
2032        ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2033        if (ret) {
2034                dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2035                goto err_free;
2036        }
2037
2038        ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2039                                        thunderx_l2c_isr,
2040                                        thunderx_l2c_threaded_isr,
2041                                        0, "[EDAC] ThunderX L2C",
2042                                        &l2c->msix_ent);
2043        if (ret)
2044                goto err_free;
2045
2046        edac_dev->dev = &pdev->dev;
2047        edac_dev->dev_name = dev_name(&pdev->dev);
2048        edac_dev->mod_name = "thunderx-l2c";
2049        edac_dev->ctl_name = "thunderx-l2c";
2050
2051        ret = edac_device_add_device(edac_dev);
2052        if (ret) {
2053                dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2054                goto err_free;
2055        }
2056
2057        if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2058                l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2059
2060                ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2061                                              l2c, dfs_entries);
2062
2063                if (ret != dfs_entries) {
2064                        dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2065                                 ret, ret >= 0 ? " created" : "");
2066                }
2067        }
2068
2069        pci_set_drvdata(pdev, edac_dev);
2070
2071        writeq(reg_en_mask, l2c->regs + reg_en_offs);
2072
2073        return 0;
2074
2075err_free:
2076        edac_device_free_ctl_info(edac_dev);
2077
2078        return ret;
2079}
2080
2081static void thunderx_l2c_remove(struct pci_dev *pdev)
2082{
2083        struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2084        struct thunderx_l2c *l2c = edac_dev->pvt_info;
2085
2086        switch (pdev->device) {
2087        case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2088                writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2089                break;
2090        case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2091                writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2092                break;
2093        case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2094                writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2095                break;
2096        }
2097
2098        edac_debugfs_remove_recursive(l2c->debugfs);
2099
2100        edac_device_del_device(&pdev->dev);
2101        edac_device_free_ctl_info(edac_dev);
2102}
2103
2104MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2105
2106static struct pci_driver thunderx_l2c_driver = {
2107        .name     = "thunderx_l2c_edac",
2108        .probe    = thunderx_l2c_probe,
2109        .remove   = thunderx_l2c_remove,
2110        .id_table = thunderx_l2c_pci_tbl,
2111};
2112
2113static int __init thunderx_edac_init(void)
2114{
2115        int rc = 0;
2116
2117        rc = pci_register_driver(&thunderx_lmc_driver);
2118        if (rc)
2119                return rc;
2120
2121        rc = pci_register_driver(&thunderx_ocx_driver);
2122        if (rc)
2123                goto err_lmc;
2124
2125        rc = pci_register_driver(&thunderx_l2c_driver);
2126        if (rc)
2127                goto err_ocx;
2128
2129        return rc;
2130err_ocx:
2131        pci_unregister_driver(&thunderx_ocx_driver);
2132err_lmc:
2133        pci_unregister_driver(&thunderx_lmc_driver);
2134
2135        return rc;
2136}
2137
2138static void __exit thunderx_edac_exit(void)
2139{
2140        pci_unregister_driver(&thunderx_l2c_driver);
2141        pci_unregister_driver(&thunderx_ocx_driver);
2142        pci_unregister_driver(&thunderx_lmc_driver);
2143
2144}
2145
2146module_init(thunderx_edac_init);
2147module_exit(thunderx_edac_exit);
2148
2149MODULE_LICENSE("GPL v2");
2150MODULE_AUTHOR("Cavium, Inc.");
2151MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");
2152