linux/drivers/edac/i10nm_base.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Driver for Intel(R) 10nm server memory controller.
   4 * Copyright (c) 2019, Intel Corporation.
   5 *
   6 */
   7
   8#include <linux/kernel.h>
   9#include <linux/io.h>
  10#include <asm/cpu_device_id.h>
  11#include <asm/intel-family.h>
  12#include <asm/mce.h>
  13#include "edac_module.h"
  14#include "skx_common.h"
  15
  16#define I10NM_REVISION  "v0.0.5"
  17#define EDAC_MOD_STR    "i10nm_edac"
  18
  19/* Debug macros */
  20#define i10nm_printk(level, fmt, arg...)        \
  21        edac_printk(level, "i10nm", fmt, ##arg)
  22
  23#define I10NM_GET_SCK_BAR(d, reg)       \
  24        pci_read_config_dword((d)->uracu, 0xd0, &(reg))
  25#define I10NM_GET_IMC_BAR(d, i, reg)    \
  26        pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg))
  27#define I10NM_GET_SAD(d, offset, i, reg)\
  28        pci_read_config_dword((d)->sad_all, (offset) + (i) * 8, &(reg))
  29#define I10NM_GET_HBM_IMC_BAR(d, reg)   \
  30        pci_read_config_dword((d)->uracu, 0xd4, &(reg))
  31#define I10NM_GET_CAPID3_CFG(d, reg)    \
  32        pci_read_config_dword((d)->pcu_cr3, 0x90, &(reg))
  33#define I10NM_GET_DIMMMTR(m, i, j)      \
  34        readl((m)->mbase + ((m)->hbm_mc ? 0x80c : 0x2080c) + \
  35        (i) * (m)->chan_mmio_sz + (j) * 4)
  36#define I10NM_GET_MCDDRTCFG(m, i, j)    \
  37        readl((m)->mbase + ((m)->hbm_mc ? 0x970 : 0x20970) + \
  38        (i) * (m)->chan_mmio_sz + (j) * 4)
  39#define I10NM_GET_MCMTR(m, i)           \
  40        readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : 0x20ef8) + \
  41        (i) * (m)->chan_mmio_sz)
  42#define I10NM_GET_AMAP(m, i)            \
  43        readl((m)->mbase + ((m)->hbm_mc ? 0x814 : 0x20814) + \
  44        (i) * (m)->chan_mmio_sz)
  45
  46#define I10NM_GET_SCK_MMIO_BASE(reg)    (GET_BITFIELD(reg, 0, 28) << 23)
  47#define I10NM_GET_IMC_MMIO_OFFSET(reg)  (GET_BITFIELD(reg, 0, 10) << 12)
  48#define I10NM_GET_IMC_MMIO_SIZE(reg)    ((GET_BITFIELD(reg, 13, 23) - \
  49                                         GET_BITFIELD(reg, 0, 10) + 1) << 12)
  50#define I10NM_GET_HBM_IMC_MMIO_OFFSET(reg)      \
  51        ((GET_BITFIELD(reg, 0, 10) << 12) + 0x140000)
  52
  53#define I10NM_HBM_IMC_MMIO_SIZE         0x9000
  54#define I10NM_IS_HBM_PRESENT(reg)       GET_BITFIELD(reg, 27, 30)
  55#define I10NM_IS_HBM_IMC(reg)           GET_BITFIELD(reg, 29, 29)
  56
  57#define I10NM_MAX_SAD                   16
  58#define I10NM_SAD_ENABLE(reg)           GET_BITFIELD(reg, 0, 0)
  59#define I10NM_SAD_NM_CACHEABLE(reg)     GET_BITFIELD(reg, 5, 5)
  60
  61static struct list_head *i10nm_edac_list;
  62
  63static struct pci_dev *pci_get_dev_wrapper(int dom, unsigned int bus,
  64                                           unsigned int dev, unsigned int fun)
  65{
  66        struct pci_dev *pdev;
  67
  68        pdev = pci_get_domain_bus_and_slot(dom, bus, PCI_DEVFN(dev, fun));
  69        if (!pdev) {
  70                edac_dbg(2, "No device %02x:%02x.%x\n",
  71                         bus, dev, fun);
  72                return NULL;
  73        }
  74
  75        if (unlikely(pci_enable_device(pdev) < 0)) {
  76                edac_dbg(2, "Failed to enable device %02x:%02x.%x\n",
  77                         bus, dev, fun);
  78                return NULL;
  79        }
  80
  81        pci_dev_get(pdev);
  82
  83        return pdev;
  84}
  85
  86static bool i10nm_check_2lm(struct res_config *cfg)
  87{
  88        struct skx_dev *d;
  89        u32 reg;
  90        int i;
  91
  92        list_for_each_entry(d, i10nm_edac_list, list) {
  93                d->sad_all = pci_get_dev_wrapper(d->seg, d->bus[1],
  94                                                 PCI_SLOT(cfg->sad_all_devfn),
  95                                                 PCI_FUNC(cfg->sad_all_devfn));
  96                if (!d->sad_all)
  97                        continue;
  98
  99                for (i = 0; i < I10NM_MAX_SAD; i++) {
 100                        I10NM_GET_SAD(d, cfg->sad_all_offset, i, reg);
 101                        if (I10NM_SAD_ENABLE(reg) && I10NM_SAD_NM_CACHEABLE(reg)) {
 102                                edac_dbg(2, "2-level memory configuration.\n");
 103                                return true;
 104                        }
 105                }
 106        }
 107
 108        return false;
 109}
 110
 111static int i10nm_get_ddr_munits(void)
 112{
 113        struct pci_dev *mdev;
 114        void __iomem *mbase;
 115        unsigned long size;
 116        struct skx_dev *d;
 117        int i, j = 0;
 118        u32 reg, off;
 119        u64 base;
 120
 121        list_for_each_entry(d, i10nm_edac_list, list) {
 122                d->util_all = pci_get_dev_wrapper(d->seg, d->bus[1], 29, 1);
 123                if (!d->util_all)
 124                        return -ENODEV;
 125
 126                d->uracu = pci_get_dev_wrapper(d->seg, d->bus[0], 0, 1);
 127                if (!d->uracu)
 128                        return -ENODEV;
 129
 130                if (I10NM_GET_SCK_BAR(d, reg)) {
 131                        i10nm_printk(KERN_ERR, "Failed to socket bar\n");
 132                        return -ENODEV;
 133                }
 134
 135                base = I10NM_GET_SCK_MMIO_BASE(reg);
 136                edac_dbg(2, "socket%d mmio base 0x%llx (reg 0x%x)\n",
 137                         j++, base, reg);
 138
 139                for (i = 0; i < I10NM_NUM_DDR_IMC; i++) {
 140                        mdev = pci_get_dev_wrapper(d->seg, d->bus[0],
 141                                                   12 + i, 0);
 142                        if (i == 0 && !mdev) {
 143                                i10nm_printk(KERN_ERR, "No IMC found\n");
 144                                return -ENODEV;
 145                        }
 146                        if (!mdev)
 147                                continue;
 148
 149                        d->imc[i].mdev = mdev;
 150
 151                        if (I10NM_GET_IMC_BAR(d, i, reg)) {
 152                                i10nm_printk(KERN_ERR, "Failed to get mc bar\n");
 153                                return -ENODEV;
 154                        }
 155
 156                        off  = I10NM_GET_IMC_MMIO_OFFSET(reg);
 157                        size = I10NM_GET_IMC_MMIO_SIZE(reg);
 158                        edac_dbg(2, "mc%d mmio base 0x%llx size 0x%lx (reg 0x%x)\n",
 159                                 i, base + off, size, reg);
 160
 161                        mbase = ioremap(base + off, size);
 162                        if (!mbase) {
 163                                i10nm_printk(KERN_ERR, "Failed to ioremap 0x%llx\n",
 164                                             base + off);
 165                                return -ENODEV;
 166                        }
 167
 168                        d->imc[i].mbase = mbase;
 169                }
 170        }
 171
 172        return 0;
 173}
 174
 175static bool i10nm_check_hbm_imc(struct skx_dev *d)
 176{
 177        u32 reg;
 178
 179        if (I10NM_GET_CAPID3_CFG(d, reg)) {
 180                i10nm_printk(KERN_ERR, "Failed to get capid3_cfg\n");
 181                return false;
 182        }
 183
 184        return I10NM_IS_HBM_PRESENT(reg) != 0;
 185}
 186
 187static int i10nm_get_hbm_munits(void)
 188{
 189        struct pci_dev *mdev;
 190        void __iomem *mbase;
 191        u32 reg, off, mcmtr;
 192        struct skx_dev *d;
 193        int i, lmc;
 194        u64 base;
 195
 196        list_for_each_entry(d, i10nm_edac_list, list) {
 197                d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[1], 30, 3);
 198                if (!d->pcu_cr3)
 199                        return -ENODEV;
 200
 201                if (!i10nm_check_hbm_imc(d)) {
 202                        i10nm_printk(KERN_DEBUG, "No hbm memory\n");
 203                        return -ENODEV;
 204                }
 205
 206                if (I10NM_GET_SCK_BAR(d, reg)) {
 207                        i10nm_printk(KERN_ERR, "Failed to get socket bar\n");
 208                        return -ENODEV;
 209                }
 210                base = I10NM_GET_SCK_MMIO_BASE(reg);
 211
 212                if (I10NM_GET_HBM_IMC_BAR(d, reg)) {
 213                        i10nm_printk(KERN_ERR, "Failed to get hbm mc bar\n");
 214                        return -ENODEV;
 215                }
 216                base += I10NM_GET_HBM_IMC_MMIO_OFFSET(reg);
 217
 218                lmc = I10NM_NUM_DDR_IMC;
 219
 220                for (i = 0; i < I10NM_NUM_HBM_IMC; i++) {
 221                        mdev = pci_get_dev_wrapper(d->seg, d->bus[0],
 222                                                   12 + i / 4, 1 + i % 4);
 223                        if (i == 0 && !mdev) {
 224                                i10nm_printk(KERN_ERR, "No hbm mc found\n");
 225                                return -ENODEV;
 226                        }
 227                        if (!mdev)
 228                                continue;
 229
 230                        d->imc[lmc].mdev = mdev;
 231                        off = i * I10NM_HBM_IMC_MMIO_SIZE;
 232
 233                        edac_dbg(2, "hbm mc%d mmio base 0x%llx size 0x%x\n",
 234                                 lmc, base + off, I10NM_HBM_IMC_MMIO_SIZE);
 235
 236                        mbase = ioremap(base + off, I10NM_HBM_IMC_MMIO_SIZE);
 237                        if (!mbase) {
 238                                i10nm_printk(KERN_ERR, "Failed to ioremap for hbm mc 0x%llx\n",
 239                                             base + off);
 240                                return -ENOMEM;
 241                        }
 242
 243                        d->imc[lmc].mbase = mbase;
 244                        d->imc[lmc].hbm_mc = true;
 245
 246                        mcmtr = I10NM_GET_MCMTR(&d->imc[lmc], 0);
 247                        if (!I10NM_IS_HBM_IMC(mcmtr)) {
 248                                i10nm_printk(KERN_ERR, "This isn't an hbm mc!\n");
 249                                return -ENODEV;
 250                        }
 251
 252                        lmc++;
 253                }
 254        }
 255
 256        return 0;
 257}
 258
 259static struct res_config i10nm_cfg0 = {
 260        .type                   = I10NM,
 261        .decs_did               = 0x3452,
 262        .busno_cfg_offset       = 0xcc,
 263        .ddr_chan_mmio_sz       = 0x4000,
 264        .sad_all_devfn          = PCI_DEVFN(29, 0),
 265        .sad_all_offset         = 0x108,
 266};
 267
 268static struct res_config i10nm_cfg1 = {
 269        .type                   = I10NM,
 270        .decs_did               = 0x3452,
 271        .busno_cfg_offset       = 0xd0,
 272        .ddr_chan_mmio_sz       = 0x4000,
 273        .sad_all_devfn          = PCI_DEVFN(29, 0),
 274        .sad_all_offset         = 0x108,
 275};
 276
 277static struct res_config spr_cfg = {
 278        .type                   = SPR,
 279        .decs_did               = 0x3252,
 280        .busno_cfg_offset       = 0xd0,
 281        .ddr_chan_mmio_sz       = 0x8000,
 282        .hbm_chan_mmio_sz       = 0x4000,
 283        .support_ddr5           = true,
 284        .sad_all_devfn          = PCI_DEVFN(10, 0),
 285        .sad_all_offset         = 0x300,
 286};
 287
 288static const struct x86_cpu_id i10nm_cpuids[] = {
 289        X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D,    X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0),
 290        X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D,    X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1),
 291        X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X,         X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0),
 292        X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X,         X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1),
 293        X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D,         X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1),
 294        X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X,  X86_STEPPINGS(0x0, 0xf), &spr_cfg),
 295        {}
 296};
 297MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids);
 298
 299static bool i10nm_check_ecc(struct skx_imc *imc, int chan)
 300{
 301        u32 mcmtr;
 302
 303        mcmtr = I10NM_GET_MCMTR(imc, chan);
 304        edac_dbg(1, "ch%d mcmtr reg %x\n", chan, mcmtr);
 305
 306        return !!GET_BITFIELD(mcmtr, 2, 2);
 307}
 308
 309static int i10nm_get_dimm_config(struct mem_ctl_info *mci,
 310                                 struct res_config *cfg)
 311{
 312        struct skx_pvt *pvt = mci->pvt_info;
 313        struct skx_imc *imc = pvt->imc;
 314        u32 mtr, amap, mcddrtcfg;
 315        struct dimm_info *dimm;
 316        int i, j, ndimms;
 317
 318        for (i = 0; i < imc->num_channels; i++) {
 319                if (!imc->mbase)
 320                        continue;
 321
 322                ndimms = 0;
 323                amap = I10NM_GET_AMAP(imc, i);
 324                for (j = 0; j < imc->num_dimms; j++) {
 325                        dimm = edac_get_dimm(mci, i, j, 0);
 326                        mtr = I10NM_GET_DIMMMTR(imc, i, j);
 327                        mcddrtcfg = I10NM_GET_MCDDRTCFG(imc, i, j);
 328                        edac_dbg(1, "dimmmtr 0x%x mcddrtcfg 0x%x (mc%d ch%d dimm%d)\n",
 329                                 mtr, mcddrtcfg, imc->mc, i, j);
 330
 331                        if (IS_DIMM_PRESENT(mtr))
 332                                ndimms += skx_get_dimm_info(mtr, 0, amap, dimm,
 333                                                            imc, i, j, cfg);
 334                        else if (IS_NVDIMM_PRESENT(mcddrtcfg, j))
 335                                ndimms += skx_get_nvdimm_info(dimm, imc, i, j,
 336                                                              EDAC_MOD_STR);
 337                }
 338                if (ndimms && !i10nm_check_ecc(imc, i)) {
 339                        i10nm_printk(KERN_ERR, "ECC is disabled on imc %d channel %d\n",
 340                                     imc->mc, i);
 341                        return -ENODEV;
 342                }
 343        }
 344
 345        return 0;
 346}
 347
 348static struct notifier_block i10nm_mce_dec = {
 349        .notifier_call  = skx_mce_check_error,
 350        .priority       = MCE_PRIO_EDAC,
 351};
 352
 353#ifdef CONFIG_EDAC_DEBUG
 354/*
 355 * Debug feature.
 356 * Exercise the address decode logic by writing an address to
 357 * /sys/kernel/debug/edac/i10nm_test/addr.
 358 */
 359static struct dentry *i10nm_test;
 360
 361static int debugfs_u64_set(void *data, u64 val)
 362{
 363        struct mce m;
 364
 365        pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
 366
 367        memset(&m, 0, sizeof(m));
 368        /* ADDRV + MemRd + Unknown channel */
 369        m.status = MCI_STATUS_ADDRV + 0x90;
 370        /* One corrected error */
 371        m.status |= BIT_ULL(MCI_STATUS_CEC_SHIFT);
 372        m.addr = val;
 373        skx_mce_check_error(NULL, 0, &m);
 374
 375        return 0;
 376}
 377DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
 378
 379static void setup_i10nm_debug(void)
 380{
 381        i10nm_test = edac_debugfs_create_dir("i10nm_test");
 382        if (!i10nm_test)
 383                return;
 384
 385        if (!edac_debugfs_create_file("addr", 0200, i10nm_test,
 386                                      NULL, &fops_u64_wo)) {
 387                debugfs_remove(i10nm_test);
 388                i10nm_test = NULL;
 389        }
 390}
 391
 392static void teardown_i10nm_debug(void)
 393{
 394        debugfs_remove_recursive(i10nm_test);
 395}
 396#else
 397static inline void setup_i10nm_debug(void) {}
 398static inline void teardown_i10nm_debug(void) {}
 399#endif /*CONFIG_EDAC_DEBUG*/
 400
 401static int __init i10nm_init(void)
 402{
 403        u8 mc = 0, src_id = 0, node_id = 0;
 404        const struct x86_cpu_id *id;
 405        struct res_config *cfg;
 406        const char *owner;
 407        struct skx_dev *d;
 408        int rc, i, off[3] = {0xd0, 0xc8, 0xcc};
 409        u64 tolm, tohm;
 410
 411        edac_dbg(2, "\n");
 412
 413        owner = edac_get_owner();
 414        if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
 415                return -EBUSY;
 416
 417        if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
 418                return -ENODEV;
 419
 420        id = x86_match_cpu(i10nm_cpuids);
 421        if (!id)
 422                return -ENODEV;
 423
 424        cfg = (struct res_config *)id->driver_data;
 425
 426        rc = skx_get_hi_lo(0x09a2, off, &tolm, &tohm);
 427        if (rc)
 428                return rc;
 429
 430        rc = skx_get_all_bus_mappings(cfg, &i10nm_edac_list);
 431        if (rc < 0)
 432                goto fail;
 433        if (rc == 0) {
 434                i10nm_printk(KERN_ERR, "No memory controllers found\n");
 435                return -ENODEV;
 436        }
 437
 438        skx_set_mem_cfg(i10nm_check_2lm(cfg));
 439
 440        rc = i10nm_get_ddr_munits();
 441
 442        if (i10nm_get_hbm_munits() && rc)
 443                goto fail;
 444
 445        list_for_each_entry(d, i10nm_edac_list, list) {
 446                rc = skx_get_src_id(d, 0xf8, &src_id);
 447                if (rc < 0)
 448                        goto fail;
 449
 450                rc = skx_get_node_id(d, &node_id);
 451                if (rc < 0)
 452                        goto fail;
 453
 454                edac_dbg(2, "src_id = %d node_id = %d\n", src_id, node_id);
 455                for (i = 0; i < I10NM_NUM_IMC; i++) {
 456                        if (!d->imc[i].mdev)
 457                                continue;
 458
 459                        d->imc[i].mc  = mc++;
 460                        d->imc[i].lmc = i;
 461                        d->imc[i].src_id  = src_id;
 462                        d->imc[i].node_id = node_id;
 463                        if (d->imc[i].hbm_mc) {
 464                                d->imc[i].chan_mmio_sz = cfg->hbm_chan_mmio_sz;
 465                                d->imc[i].num_channels = I10NM_NUM_HBM_CHANNELS;
 466                                d->imc[i].num_dimms    = I10NM_NUM_HBM_DIMMS;
 467                        } else {
 468                                d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz;
 469                                d->imc[i].num_channels = I10NM_NUM_DDR_CHANNELS;
 470                                d->imc[i].num_dimms    = I10NM_NUM_DDR_DIMMS;
 471                        }
 472
 473                        rc = skx_register_mci(&d->imc[i], d->imc[i].mdev,
 474                                              "Intel_10nm Socket", EDAC_MOD_STR,
 475                                              i10nm_get_dimm_config, cfg);
 476                        if (rc < 0)
 477                                goto fail;
 478                }
 479        }
 480
 481        rc = skx_adxl_get();
 482        if (rc)
 483                goto fail;
 484
 485        opstate_init();
 486        mce_register_decode_chain(&i10nm_mce_dec);
 487        setup_i10nm_debug();
 488
 489        i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION);
 490
 491        return 0;
 492fail:
 493        skx_remove();
 494        return rc;
 495}
 496
 497static void __exit i10nm_exit(void)
 498{
 499        edac_dbg(2, "\n");
 500        teardown_i10nm_debug();
 501        mce_unregister_decode_chain(&i10nm_mce_dec);
 502        skx_adxl_put();
 503        skx_remove();
 504}
 505
 506module_init(i10nm_init);
 507module_exit(i10nm_exit);
 508
 509MODULE_LICENSE("GPL v2");
 510MODULE_DESCRIPTION("MC Driver for Intel 10nm server processors");
 511