linux/drivers/edac/sb_edac.c
<<
>>
Prefs
   1/* Intel Sandy Bridge -EN/-EP/-EX Memory Controller kernel module
   2 *
   3 * This driver supports the memory controllers found on the Intel
   4 * processor family Sandy Bridge.
   5 *
   6 * This file may be distributed under the terms of the
   7 * GNU General Public License version 2 only.
   8 *
   9 * Copyright (c) 2011 by:
  10 *       Mauro Carvalho Chehab <mchehab@redhat.com>
  11 */
  12
  13#include <linux/module.h>
  14#include <linux/init.h>
  15#include <linux/pci.h>
  16#include <linux/pci_ids.h>
  17#include <linux/slab.h>
  18#include <linux/delay.h>
  19#include <linux/edac.h>
  20#include <linux/mmzone.h>
  21#include <linux/smp.h>
  22#include <linux/bitmap.h>
  23#include <linux/math64.h>
  24#include <asm/processor.h>
  25#include <asm/mce.h>
  26
  27#include "edac_core.h"
  28
  29/* Static vars */
  30static LIST_HEAD(sbridge_edac_list);
  31static DEFINE_MUTEX(sbridge_edac_lock);
  32static int probed;
  33
  34/*
  35 * Alter this version for the module when modifications are made
  36 */
  37#define SBRIDGE_REVISION    " Ver: 1.0.0 "
  38#define EDAC_MOD_STR      "sbridge_edac"
  39
  40/*
  41 * Debug macros
  42 */
  43#define sbridge_printk(level, fmt, arg...)                      \
  44        edac_printk(level, "sbridge", fmt, ##arg)
  45
  46#define sbridge_mc_printk(mci, level, fmt, arg...)              \
  47        edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg)
  48
  49/*
  50 * Get a bit field at register value <v>, from bit <lo> to bit <hi>
  51 */
  52#define GET_BITFIELD(v, lo, hi) \
  53        (((v) & ((1ULL << ((hi) - (lo) + 1)) - 1) << (lo)) >> (lo))
  54
  55/*
  56 * sbridge Memory Controller Registers
  57 */
  58
  59/*
  60 * FIXME: For now, let's order by device function, as it makes
  61 * easier for driver's development process. This table should be
  62 * moved to pci_id.h when submitted upstream
  63 */
  64#define PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0        0x3cf4  /* 12.6 */
  65#define PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1        0x3cf6  /* 12.7 */
  66#define PCI_DEVICE_ID_INTEL_SBRIDGE_BR          0x3cf5  /* 13.6 */
  67#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0     0x3ca0  /* 14.0 */
  68#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA      0x3ca8  /* 15.0 */
  69#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS     0x3c71  /* 15.1 */
  70#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0    0x3caa  /* 15.2 */
  71#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1    0x3cab  /* 15.3 */
  72#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2    0x3cac  /* 15.4 */
  73#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3    0x3cad  /* 15.5 */
  74#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO   0x3cb8  /* 17.0 */
  75
  76        /*
  77         * Currently, unused, but will be needed in the future
  78         * implementations, as they hold the error counters
  79         */
  80#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR0    0x3c72  /* 16.2 */
  81#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR1    0x3c73  /* 16.3 */
  82#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR2    0x3c76  /* 16.6 */
  83#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR3    0x3c77  /* 16.7 */
  84
  85/* Devices 12 Function 6, Offsets 0x80 to 0xcc */
  86static const u32 dram_rule[] = {
  87        0x80, 0x88, 0x90, 0x98, 0xa0,
  88        0xa8, 0xb0, 0xb8, 0xc0, 0xc8,
  89};
  90#define MAX_SAD         ARRAY_SIZE(dram_rule)
  91
  92#define SAD_LIMIT(reg)          ((GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff)
  93#define DRAM_ATTR(reg)          GET_BITFIELD(reg, 2,  3)
  94#define INTERLEAVE_MODE(reg)    GET_BITFIELD(reg, 1,  1)
  95#define DRAM_RULE_ENABLE(reg)   GET_BITFIELD(reg, 0,  0)
  96
  97static char *get_dram_attr(u32 reg)
  98{
  99        switch(DRAM_ATTR(reg)) {
 100                case 0:
 101                        return "DRAM";
 102                case 1:
 103                        return "MMCFG";
 104                case 2:
 105                        return "NXM";
 106                default:
 107                        return "unknown";
 108        }
 109}
 110
 111static const u32 interleave_list[] = {
 112        0x84, 0x8c, 0x94, 0x9c, 0xa4,
 113        0xac, 0xb4, 0xbc, 0xc4, 0xcc,
 114};
 115#define MAX_INTERLEAVE  ARRAY_SIZE(interleave_list)
 116
 117#define SAD_PKG0(reg)           GET_BITFIELD(reg, 0, 2)
 118#define SAD_PKG1(reg)           GET_BITFIELD(reg, 3, 5)
 119#define SAD_PKG2(reg)           GET_BITFIELD(reg, 8, 10)
 120#define SAD_PKG3(reg)           GET_BITFIELD(reg, 11, 13)
 121#define SAD_PKG4(reg)           GET_BITFIELD(reg, 16, 18)
 122#define SAD_PKG5(reg)           GET_BITFIELD(reg, 19, 21)
 123#define SAD_PKG6(reg)           GET_BITFIELD(reg, 24, 26)
 124#define SAD_PKG7(reg)           GET_BITFIELD(reg, 27, 29)
 125
 126static inline int sad_pkg(u32 reg, int interleave)
 127{
 128        switch (interleave) {
 129        case 0:
 130                return SAD_PKG0(reg);
 131        case 1:
 132                return SAD_PKG1(reg);
 133        case 2:
 134                return SAD_PKG2(reg);
 135        case 3:
 136                return SAD_PKG3(reg);
 137        case 4:
 138                return SAD_PKG4(reg);
 139        case 5:
 140                return SAD_PKG5(reg);
 141        case 6:
 142                return SAD_PKG6(reg);
 143        case 7:
 144                return SAD_PKG7(reg);
 145        default:
 146                return -EINVAL;
 147        }
 148}
 149
 150/* Devices 12 Function 7 */
 151
 152#define TOLM            0x80
 153#define TOHM            0x84
 154
 155#define GET_TOLM(reg)           ((GET_BITFIELD(reg, 0,  3) << 28) | 0x3ffffff)
 156#define GET_TOHM(reg)           ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff)
 157
 158/* Device 13 Function 6 */
 159
 160#define SAD_TARGET      0xf0
 161
 162#define SOURCE_ID(reg)          GET_BITFIELD(reg, 9, 11)
 163
 164#define SAD_CONTROL     0xf4
 165
 166#define NODE_ID(reg)            GET_BITFIELD(reg, 0, 2)
 167
 168/* Device 14 function 0 */
 169
 170static const u32 tad_dram_rule[] = {
 171        0x40, 0x44, 0x48, 0x4c,
 172        0x50, 0x54, 0x58, 0x5c,
 173        0x60, 0x64, 0x68, 0x6c,
 174};
 175#define MAX_TAD ARRAY_SIZE(tad_dram_rule)
 176
 177#define TAD_LIMIT(reg)          ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff)
 178#define TAD_SOCK(reg)           GET_BITFIELD(reg, 10, 11)
 179#define TAD_CH(reg)             GET_BITFIELD(reg,  8,  9)
 180#define TAD_TGT3(reg)           GET_BITFIELD(reg,  6,  7)
 181#define TAD_TGT2(reg)           GET_BITFIELD(reg,  4,  5)
 182#define TAD_TGT1(reg)           GET_BITFIELD(reg,  2,  3)
 183#define TAD_TGT0(reg)           GET_BITFIELD(reg,  0,  1)
 184
 185/* Device 15, function 0 */
 186
 187#define MCMTR                   0x7c
 188
 189#define IS_ECC_ENABLED(mcmtr)           GET_BITFIELD(mcmtr, 2, 2)
 190#define IS_LOCKSTEP_ENABLED(mcmtr)      GET_BITFIELD(mcmtr, 1, 1)
 191#define IS_CLOSE_PG(mcmtr)              GET_BITFIELD(mcmtr, 0, 0)
 192
 193/* Device 15, function 1 */
 194
 195#define RASENABLES              0xac
 196#define IS_MIRROR_ENABLED(reg)          GET_BITFIELD(reg, 0, 0)
 197
 198/* Device 15, functions 2-5 */
 199
 200static const int mtr_regs[] = {
 201        0x80, 0x84, 0x88,
 202};
 203
 204#define RANK_DISABLE(mtr)               GET_BITFIELD(mtr, 16, 19)
 205#define IS_DIMM_PRESENT(mtr)            GET_BITFIELD(mtr, 14, 14)
 206#define RANK_CNT_BITS(mtr)              GET_BITFIELD(mtr, 12, 13)
 207#define RANK_WIDTH_BITS(mtr)            GET_BITFIELD(mtr, 2, 4)
 208#define COL_WIDTH_BITS(mtr)             GET_BITFIELD(mtr, 0, 1)
 209
 210static const u32 tad_ch_nilv_offset[] = {
 211        0x90, 0x94, 0x98, 0x9c,
 212        0xa0, 0xa4, 0xa8, 0xac,
 213        0xb0, 0xb4, 0xb8, 0xbc,
 214};
 215#define CHN_IDX_OFFSET(reg)             GET_BITFIELD(reg, 28, 29)
 216#define TAD_OFFSET(reg)                 (GET_BITFIELD(reg,  6, 25) << 26)
 217
 218static const u32 rir_way_limit[] = {
 219        0x108, 0x10c, 0x110, 0x114, 0x118,
 220};
 221#define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit)
 222
 223#define IS_RIR_VALID(reg)       GET_BITFIELD(reg, 31, 31)
 224#define RIR_WAY(reg)            GET_BITFIELD(reg, 28, 29)
 225#define RIR_LIMIT(reg)          ((GET_BITFIELD(reg,  1, 10) << 29)| 0x1fffffff)
 226
 227#define MAX_RIR_WAY     8
 228
 229static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = {
 230        { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c },
 231        { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c },
 232        { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c },
 233        { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c },
 234        { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc },
 235};
 236
 237#define RIR_RNK_TGT(reg)                GET_BITFIELD(reg, 16, 19)
 238#define RIR_OFFSET(reg)         GET_BITFIELD(reg,  2, 14)
 239
 240/* Device 16, functions 2-7 */
 241
 242/*
 243 * FIXME: Implement the error count reads directly
 244 */
 245
 246static const u32 correrrcnt[] = {
 247        0x104, 0x108, 0x10c, 0x110,
 248};
 249
 250#define RANK_ODD_OV(reg)                GET_BITFIELD(reg, 31, 31)
 251#define RANK_ODD_ERR_CNT(reg)           GET_BITFIELD(reg, 16, 30)
 252#define RANK_EVEN_OV(reg)               GET_BITFIELD(reg, 15, 15)
 253#define RANK_EVEN_ERR_CNT(reg)          GET_BITFIELD(reg,  0, 14)
 254
 255static const u32 correrrthrsld[] = {
 256        0x11c, 0x120, 0x124, 0x128,
 257};
 258
 259#define RANK_ODD_ERR_THRSLD(reg)        GET_BITFIELD(reg, 16, 30)
 260#define RANK_EVEN_ERR_THRSLD(reg)       GET_BITFIELD(reg,  0, 14)
 261
 262
 263/* Device 17, function 0 */
 264
 265#define RANK_CFG_A              0x0328
 266
 267#define IS_RDIMM_ENABLED(reg)           GET_BITFIELD(reg, 11, 11)
 268
 269/*
 270 * sbridge structs
 271 */
 272
 273#define NUM_CHANNELS    4
 274#define MAX_DIMMS       3               /* Max DIMMS per channel */
 275
 276struct sbridge_info {
 277        u32     mcmtr;
 278};
 279
 280struct sbridge_channel {
 281        u32             ranks;
 282        u32             dimms;
 283};
 284
 285struct pci_id_descr {
 286        int                     dev;
 287        int                     func;
 288        int                     dev_id;
 289        int                     optional;
 290};
 291
 292struct pci_id_table {
 293        const struct pci_id_descr       *descr;
 294        int                             n_devs;
 295};
 296
 297struct sbridge_dev {
 298        struct list_head        list;
 299        u8                      bus, mc;
 300        u8                      node_id, source_id;
 301        struct pci_dev          **pdev;
 302        int                     n_devs;
 303        struct mem_ctl_info     *mci;
 304};
 305
 306struct sbridge_pvt {
 307        struct pci_dev          *pci_ta, *pci_ddrio, *pci_ras;
 308        struct pci_dev          *pci_sad0, *pci_sad1, *pci_ha0;
 309        struct pci_dev          *pci_br;
 310        struct pci_dev          *pci_tad[NUM_CHANNELS];
 311
 312        struct sbridge_dev      *sbridge_dev;
 313
 314        struct sbridge_info     info;
 315        struct sbridge_channel  channel[NUM_CHANNELS];
 316
 317        /* Memory type detection */
 318        bool                    is_mirrored, is_lockstep, is_close_pg;
 319
 320        /* Fifo double buffers */
 321        struct mce              mce_entry[MCE_LOG_LEN];
 322        struct mce              mce_outentry[MCE_LOG_LEN];
 323
 324        /* Fifo in/out counters */
 325        unsigned                mce_in, mce_out;
 326
 327        /* Count indicator to show errors not got */
 328        unsigned                mce_overrun;
 329
 330        /* Memory description */
 331        u64                     tolm, tohm;
 332};
 333
 334#define PCI_DESCR(device, function, device_id, opt)     \
 335        .dev = (device),                                \
 336        .func = (function),                             \
 337        .dev_id = (device_id),                          \
 338        .optional = opt
 339
 340static const struct pci_id_descr pci_dev_descr_sbridge[] = {
 341                /* Processor Home Agent */
 342        { PCI_DESCR(14, 0, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0, 0)      },
 343
 344                /* Memory controller */
 345        { PCI_DESCR(15, 0, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA, 0)       },
 346        { PCI_DESCR(15, 1, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS, 0)      },
 347        { PCI_DESCR(15, 2, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0, 0)     },
 348        { PCI_DESCR(15, 3, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1, 0)     },
 349        { PCI_DESCR(15, 4, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2, 0)     },
 350        { PCI_DESCR(15, 5, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3, 0)     },
 351        { PCI_DESCR(17, 0, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO, 1)    },
 352
 353                /* System Address Decoder */
 354        { PCI_DESCR(12, 6, PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0, 0)         },
 355        { PCI_DESCR(12, 7, PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1, 0)         },
 356
 357                /* Broadcast Registers */
 358        { PCI_DESCR(13, 6, PCI_DEVICE_ID_INTEL_SBRIDGE_BR, 0)           },
 359};
 360
 361#define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 362static const struct pci_id_table pci_dev_descr_sbridge_table[] = {
 363        PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge),
 364        {0,}                    /* 0 terminated list. */
 365};
 366
 367/*
 368 *      pci_device_id   table for which devices we are looking for
 369 */
 370static DEFINE_PCI_DEVICE_TABLE(sbridge_pci_tbl) = {
 371        {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA)},
 372        {0,}                    /* 0 terminated list. */
 373};
 374
 375
 376/****************************************************************************
 377                        Ancillary status routines
 378 ****************************************************************************/
 379
 380static inline int numrank(u32 mtr)
 381{
 382        int ranks = (1 << RANK_CNT_BITS(mtr));
 383
 384        if (ranks > 4) {
 385                edac_dbg(0, "Invalid number of ranks: %d (max = 4) raw value = %x (%04x)\n",
 386                         ranks, (unsigned int)RANK_CNT_BITS(mtr), mtr);
 387                return -EINVAL;
 388        }
 389
 390        return ranks;
 391}
 392
 393static inline int numrow(u32 mtr)
 394{
 395        int rows = (RANK_WIDTH_BITS(mtr) + 12);
 396
 397        if (rows < 13 || rows > 18) {
 398                edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n",
 399                         rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr);
 400                return -EINVAL;
 401        }
 402
 403        return 1 << rows;
 404}
 405
 406static inline int numcol(u32 mtr)
 407{
 408        int cols = (COL_WIDTH_BITS(mtr) + 10);
 409
 410        if (cols > 12) {
 411                edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n",
 412                         cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr);
 413                return -EINVAL;
 414        }
 415
 416        return 1 << cols;
 417}
 418
 419static struct sbridge_dev *get_sbridge_dev(u8 bus)
 420{
 421        struct sbridge_dev *sbridge_dev;
 422
 423        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
 424                if (sbridge_dev->bus == bus)
 425                        return sbridge_dev;
 426        }
 427
 428        return NULL;
 429}
 430
 431static struct sbridge_dev *alloc_sbridge_dev(u8 bus,
 432                                           const struct pci_id_table *table)
 433{
 434        struct sbridge_dev *sbridge_dev;
 435
 436        sbridge_dev = kzalloc(sizeof(*sbridge_dev), GFP_KERNEL);
 437        if (!sbridge_dev)
 438                return NULL;
 439
 440        sbridge_dev->pdev = kzalloc(sizeof(*sbridge_dev->pdev) * table->n_devs,
 441                                   GFP_KERNEL);
 442        if (!sbridge_dev->pdev) {
 443                kfree(sbridge_dev);
 444                return NULL;
 445        }
 446
 447        sbridge_dev->bus = bus;
 448        sbridge_dev->n_devs = table->n_devs;
 449        list_add_tail(&sbridge_dev->list, &sbridge_edac_list);
 450
 451        return sbridge_dev;
 452}
 453
 454static void free_sbridge_dev(struct sbridge_dev *sbridge_dev)
 455{
 456        list_del(&sbridge_dev->list);
 457        kfree(sbridge_dev->pdev);
 458        kfree(sbridge_dev);
 459}
 460
 461/****************************************************************************
 462                        Memory check routines
 463 ****************************************************************************/
 464static struct pci_dev *get_pdev_slot_func(u8 bus, unsigned slot,
 465                                          unsigned func)
 466{
 467        struct sbridge_dev *sbridge_dev = get_sbridge_dev(bus);
 468        int i;
 469
 470        if (!sbridge_dev)
 471                return NULL;
 472
 473        for (i = 0; i < sbridge_dev->n_devs; i++) {
 474                if (!sbridge_dev->pdev[i])
 475                        continue;
 476
 477                if (PCI_SLOT(sbridge_dev->pdev[i]->devfn) == slot &&
 478                    PCI_FUNC(sbridge_dev->pdev[i]->devfn) == func) {
 479                        edac_dbg(1, "Associated %02x.%02x.%d with %p\n",
 480                                 bus, slot, func, sbridge_dev->pdev[i]);
 481                        return sbridge_dev->pdev[i];
 482                }
 483        }
 484
 485        return NULL;
 486}
 487
 488/**
 489 * check_if_ecc_is_active() - Checks if ECC is active
 490 * bus:         Device bus
 491 */
 492static int check_if_ecc_is_active(const u8 bus)
 493{
 494        struct pci_dev *pdev = NULL;
 495        u32 mcmtr;
 496
 497        pdev = get_pdev_slot_func(bus, 15, 0);
 498        if (!pdev) {
 499                sbridge_printk(KERN_ERR, "Couldn't find PCI device "
 500                                        "%2x.%02d.%d!!!\n",
 501                                        bus, 15, 0);
 502                return -ENODEV;
 503        }
 504
 505        pci_read_config_dword(pdev, MCMTR, &mcmtr);
 506        if (!IS_ECC_ENABLED(mcmtr)) {
 507                sbridge_printk(KERN_ERR, "ECC is disabled. Aborting\n");
 508                return -ENODEV;
 509        }
 510        return 0;
 511}
 512
 513static int get_dimm_config(struct mem_ctl_info *mci)
 514{
 515        struct sbridge_pvt *pvt = mci->pvt_info;
 516        struct dimm_info *dimm;
 517        unsigned i, j, banks, ranks, rows, cols, npages;
 518        u64 size;
 519        u32 reg;
 520        enum edac_type mode;
 521        enum mem_type mtype;
 522
 523        pci_read_config_dword(pvt->pci_br, SAD_TARGET, &reg);
 524        pvt->sbridge_dev->source_id = SOURCE_ID(reg);
 525
 526        pci_read_config_dword(pvt->pci_br, SAD_CONTROL, &reg);
 527        pvt->sbridge_dev->node_id = NODE_ID(reg);
 528        edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n",
 529                 pvt->sbridge_dev->mc,
 530                 pvt->sbridge_dev->node_id,
 531                 pvt->sbridge_dev->source_id);
 532
 533        pci_read_config_dword(pvt->pci_ras, RASENABLES, &reg);
 534        if (IS_MIRROR_ENABLED(reg)) {
 535                edac_dbg(0, "Memory mirror is enabled\n");
 536                pvt->is_mirrored = true;
 537        } else {
 538                edac_dbg(0, "Memory mirror is disabled\n");
 539                pvt->is_mirrored = false;
 540        }
 541
 542        pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr);
 543        if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) {
 544                edac_dbg(0, "Lockstep is enabled\n");
 545                mode = EDAC_S8ECD8ED;
 546                pvt->is_lockstep = true;
 547        } else {
 548                edac_dbg(0, "Lockstep is disabled\n");
 549                mode = EDAC_S4ECD4ED;
 550                pvt->is_lockstep = false;
 551        }
 552        if (IS_CLOSE_PG(pvt->info.mcmtr)) {
 553                edac_dbg(0, "address map is on closed page mode\n");
 554                pvt->is_close_pg = true;
 555        } else {
 556                edac_dbg(0, "address map is on open page mode\n");
 557                pvt->is_close_pg = false;
 558        }
 559
 560        if (pvt->pci_ddrio) {
 561                pci_read_config_dword(pvt->pci_ddrio, RANK_CFG_A, &reg);
 562                if (IS_RDIMM_ENABLED(reg)) {
 563                        /* FIXME: Can also be LRDIMM */
 564                        edac_dbg(0, "Memory is registered\n");
 565                        mtype = MEM_RDDR3;
 566                } else {
 567                        edac_dbg(0, "Memory is unregistered\n");
 568                        mtype = MEM_DDR3;
 569                }
 570        } else {
 571                edac_dbg(0, "Cannot determine memory type\n");
 572                mtype = MEM_UNKNOWN;
 573        }
 574
 575        /* On all supported DDR3 DIMM types, there are 8 banks available */
 576        banks = 8;
 577
 578        for (i = 0; i < NUM_CHANNELS; i++) {
 579                u32 mtr;
 580
 581                for (j = 0; j < ARRAY_SIZE(mtr_regs); j++) {
 582                        dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers,
 583                                       i, j, 0);
 584                        pci_read_config_dword(pvt->pci_tad[i],
 585                                              mtr_regs[j], &mtr);
 586                        edac_dbg(4, "Channel #%d  MTR%d = %x\n", i, j, mtr);
 587                        if (IS_DIMM_PRESENT(mtr)) {
 588                                pvt->channel[i].dimms++;
 589
 590                                ranks = numrank(mtr);
 591                                rows = numrow(mtr);
 592                                cols = numcol(mtr);
 593
 594                                /* DDR3 has 8 I/O banks */
 595                                size = ((u64)rows * cols * banks * ranks) >> (20 - 3);
 596                                npages = MiB_TO_PAGES(size);
 597
 598                                edac_dbg(0, "mc#%d: channel %d, dimm %d, %Ld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n",
 599                                         pvt->sbridge_dev->mc, i, j,
 600                                         size, npages,
 601                                         banks, ranks, rows, cols);
 602
 603                                dimm->nr_pages = npages;
 604                                dimm->grain = 32;
 605                                dimm->dtype = (banks == 8) ? DEV_X8 : DEV_X4;
 606                                dimm->mtype = mtype;
 607                                dimm->edac_mode = mode;
 608                                snprintf(dimm->label, sizeof(dimm->label),
 609                                         "CPU_SrcID#%u_Channel#%u_DIMM#%u",
 610                                         pvt->sbridge_dev->source_id, i, j);
 611                        }
 612                }
 613        }
 614
 615        return 0;
 616}
 617
 618static void get_memory_layout(const struct mem_ctl_info *mci)
 619{
 620        struct sbridge_pvt *pvt = mci->pvt_info;
 621        int i, j, k, n_sads, n_tads, sad_interl;
 622        u32 reg;
 623        u64 limit, prv = 0;
 624        u64 tmp_mb;
 625        u32 mb, kb;
 626        u32 rir_way;
 627
 628        /*
 629         * Step 1) Get TOLM/TOHM ranges
 630         */
 631
 632        /* Address range is 32:28 */
 633        pci_read_config_dword(pvt->pci_sad1, TOLM,
 634                              &reg);
 635        pvt->tolm = GET_TOLM(reg);
 636        tmp_mb = (1 + pvt->tolm) >> 20;
 637
 638        mb = div_u64_rem(tmp_mb, 1000, &kb);
 639        edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n", mb, kb, (u64)pvt->tolm);
 640
 641        /* Address range is already 45:25 */
 642        pci_read_config_dword(pvt->pci_sad1, TOHM,
 643                              &reg);
 644        pvt->tohm = GET_TOHM(reg);
 645        tmp_mb = (1 + pvt->tohm) >> 20;
 646
 647        mb = div_u64_rem(tmp_mb, 1000, &kb);
 648        edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n", mb, kb, (u64)pvt->tohm);
 649
 650        /*
 651         * Step 2) Get SAD range and SAD Interleave list
 652         * TAD registers contain the interleave wayness. However, it
 653         * seems simpler to just discover it indirectly, with the
 654         * algorithm bellow.
 655         */
 656        prv = 0;
 657        for (n_sads = 0; n_sads < MAX_SAD; n_sads++) {
 658                /* SAD_LIMIT Address range is 45:26 */
 659                pci_read_config_dword(pvt->pci_sad0, dram_rule[n_sads],
 660                                      &reg);
 661                limit = SAD_LIMIT(reg);
 662
 663                if (!DRAM_RULE_ENABLE(reg))
 664                        continue;
 665
 666                if (limit <= prv)
 667                        break;
 668
 669                tmp_mb = (limit + 1) >> 20;
 670                mb = div_u64_rem(tmp_mb, 1000, &kb);
 671                edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n",
 672                         n_sads,
 673                         get_dram_attr(reg),
 674                         mb, kb,
 675                         ((u64)tmp_mb) << 20L,
 676                         INTERLEAVE_MODE(reg) ? "8:6" : "[8:6]XOR[18:16]",
 677                         reg);
 678                prv = limit;
 679
 680                pci_read_config_dword(pvt->pci_sad0, interleave_list[n_sads],
 681                                      &reg);
 682                sad_interl = sad_pkg(reg, 0);
 683                for (j = 0; j < 8; j++) {
 684                        if (j > 0 && sad_interl == sad_pkg(reg, j))
 685                                break;
 686
 687                        edac_dbg(0, "SAD#%d, interleave #%d: %d\n",
 688                                 n_sads, j, sad_pkg(reg, j));
 689                }
 690        }
 691
 692        /*
 693         * Step 3) Get TAD range
 694         */
 695        prv = 0;
 696        for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
 697                pci_read_config_dword(pvt->pci_ha0, tad_dram_rule[n_tads],
 698                                      &reg);
 699                limit = TAD_LIMIT(reg);
 700                if (limit <= prv)
 701                        break;
 702                tmp_mb = (limit + 1) >> 20;
 703
 704                mb = div_u64_rem(tmp_mb, 1000, &kb);
 705                edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n",
 706                         n_tads, mb, kb,
 707                         ((u64)tmp_mb) << 20L,
 708                         (u32)TAD_SOCK(reg),
 709                         (u32)TAD_CH(reg),
 710                         (u32)TAD_TGT0(reg),
 711                         (u32)TAD_TGT1(reg),
 712                         (u32)TAD_TGT2(reg),
 713                         (u32)TAD_TGT3(reg),
 714                         reg);
 715                prv = limit;
 716        }
 717
 718        /*
 719         * Step 4) Get TAD offsets, per each channel
 720         */
 721        for (i = 0; i < NUM_CHANNELS; i++) {
 722                if (!pvt->channel[i].dimms)
 723                        continue;
 724                for (j = 0; j < n_tads; j++) {
 725                        pci_read_config_dword(pvt->pci_tad[i],
 726                                              tad_ch_nilv_offset[j],
 727                                              &reg);
 728                        tmp_mb = TAD_OFFSET(reg) >> 20;
 729                        mb = div_u64_rem(tmp_mb, 1000, &kb);
 730                        edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n",
 731                                 i, j,
 732                                 mb, kb,
 733                                 ((u64)tmp_mb) << 20L,
 734                                 reg);
 735                }
 736        }
 737
 738        /*
 739         * Step 6) Get RIR Wayness/Limit, per each channel
 740         */
 741        for (i = 0; i < NUM_CHANNELS; i++) {
 742                if (!pvt->channel[i].dimms)
 743                        continue;
 744                for (j = 0; j < MAX_RIR_RANGES; j++) {
 745                        pci_read_config_dword(pvt->pci_tad[i],
 746                                              rir_way_limit[j],
 747                                              &reg);
 748
 749                        if (!IS_RIR_VALID(reg))
 750                                continue;
 751
 752                        tmp_mb = RIR_LIMIT(reg) >> 20;
 753                        rir_way = 1 << RIR_WAY(reg);
 754                        mb = div_u64_rem(tmp_mb, 1000, &kb);
 755                        edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n",
 756                                 i, j,
 757                                 mb, kb,
 758                                 ((u64)tmp_mb) << 20L,
 759                                 rir_way,
 760                                 reg);
 761
 762                        for (k = 0; k < rir_way; k++) {
 763                                pci_read_config_dword(pvt->pci_tad[i],
 764                                                      rir_offset[j][k],
 765                                                      &reg);
 766                                tmp_mb = RIR_OFFSET(reg) << 6;
 767
 768                                mb = div_u64_rem(tmp_mb, 1000, &kb);
 769                                edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n",
 770                                         i, j, k,
 771                                         mb, kb,
 772                                         ((u64)tmp_mb) << 20L,
 773                                         (u32)RIR_RNK_TGT(reg),
 774                                         reg);
 775                        }
 776                }
 777        }
 778}
 779
 780struct mem_ctl_info *get_mci_for_node_id(u8 node_id)
 781{
 782        struct sbridge_dev *sbridge_dev;
 783
 784        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
 785                if (sbridge_dev->node_id == node_id)
 786                        return sbridge_dev->mci;
 787        }
 788        return NULL;
 789}
 790
 791static int get_memory_error_data(struct mem_ctl_info *mci,
 792                                 u64 addr,
 793                                 u8 *socket,
 794                                 long *channel_mask,
 795                                 u8 *rank,
 796                                 char **area_type, char *msg)
 797{
 798        struct mem_ctl_info     *new_mci;
 799        struct sbridge_pvt *pvt = mci->pvt_info;
 800        int                     n_rir, n_sads, n_tads, sad_way, sck_xch;
 801        int                     sad_interl, idx, base_ch;
 802        int                     interleave_mode;
 803        unsigned                sad_interleave[MAX_INTERLEAVE];
 804        u32                     reg;
 805        u8                      ch_way,sck_way;
 806        u32                     tad_offset;
 807        u32                     rir_way;
 808        u32                     mb, kb;
 809        u64                     ch_addr, offset, limit, prv = 0;
 810
 811
 812        /*
 813         * Step 0) Check if the address is at special memory ranges
 814         * The check bellow is probably enough to fill all cases where
 815         * the error is not inside a memory, except for the legacy
 816         * range (e. g. VGA addresses). It is unlikely, however, that the
 817         * memory controller would generate an error on that range.
 818         */
 819        if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) {
 820                sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr);
 821                return -EINVAL;
 822        }
 823        if (addr >= (u64)pvt->tohm) {
 824                sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr);
 825                return -EINVAL;
 826        }
 827
 828        /*
 829         * Step 1) Get socket
 830         */
 831        for (n_sads = 0; n_sads < MAX_SAD; n_sads++) {
 832                pci_read_config_dword(pvt->pci_sad0, dram_rule[n_sads],
 833                                      &reg);
 834
 835                if (!DRAM_RULE_ENABLE(reg))
 836                        continue;
 837
 838                limit = SAD_LIMIT(reg);
 839                if (limit <= prv) {
 840                        sprintf(msg, "Can't discover the memory socket");
 841                        return -EINVAL;
 842                }
 843                if  (addr <= limit)
 844                        break;
 845                prv = limit;
 846        }
 847        if (n_sads == MAX_SAD) {
 848                sprintf(msg, "Can't discover the memory socket");
 849                return -EINVAL;
 850        }
 851        *area_type = get_dram_attr(reg);
 852        interleave_mode = INTERLEAVE_MODE(reg);
 853
 854        pci_read_config_dword(pvt->pci_sad0, interleave_list[n_sads],
 855                              &reg);
 856        sad_interl = sad_pkg(reg, 0);
 857        for (sad_way = 0; sad_way < 8; sad_way++) {
 858                if (sad_way > 0 && sad_interl == sad_pkg(reg, sad_way))
 859                        break;
 860                sad_interleave[sad_way] = sad_pkg(reg, sad_way);
 861                edac_dbg(0, "SAD interleave #%d: %d\n",
 862                         sad_way, sad_interleave[sad_way]);
 863        }
 864        edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n",
 865                 pvt->sbridge_dev->mc,
 866                 n_sads,
 867                 addr,
 868                 limit,
 869                 sad_way + 7,
 870                 interleave_mode ? "" : "XOR[18:16]");
 871        if (interleave_mode)
 872                idx = ((addr >> 6) ^ (addr >> 16)) & 7;
 873        else
 874                idx = (addr >> 6) & 7;
 875        switch (sad_way) {
 876        case 1:
 877                idx = 0;
 878                break;
 879        case 2:
 880                idx = idx & 1;
 881                break;
 882        case 4:
 883                idx = idx & 3;
 884                break;
 885        case 8:
 886                break;
 887        default:
 888                sprintf(msg, "Can't discover socket interleave");
 889                return -EINVAL;
 890        }
 891        *socket = sad_interleave[idx];
 892        edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n",
 893                 idx, sad_way, *socket);
 894
 895        /*
 896         * Move to the proper node structure, in order to access the
 897         * right PCI registers
 898         */
 899        new_mci = get_mci_for_node_id(*socket);
 900        if (!new_mci) {
 901                sprintf(msg, "Struct for socket #%u wasn't initialized",
 902                        *socket);
 903                return -EINVAL;
 904        }
 905        mci = new_mci;
 906        pvt = mci->pvt_info;
 907
 908        /*
 909         * Step 2) Get memory channel
 910         */
 911        prv = 0;
 912        for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
 913                pci_read_config_dword(pvt->pci_ha0, tad_dram_rule[n_tads],
 914                                      &reg);
 915                limit = TAD_LIMIT(reg);
 916                if (limit <= prv) {
 917                        sprintf(msg, "Can't discover the memory channel");
 918                        return -EINVAL;
 919                }
 920                if  (addr <= limit)
 921                        break;
 922                prv = limit;
 923        }
 924        ch_way = TAD_CH(reg) + 1;
 925        sck_way = TAD_SOCK(reg) + 1;
 926        /*
 927         * FIXME: Is it right to always use channel 0 for offsets?
 928         */
 929        pci_read_config_dword(pvt->pci_tad[0],
 930                                tad_ch_nilv_offset[n_tads],
 931                                &tad_offset);
 932
 933        if (ch_way == 3)
 934                idx = addr >> 6;
 935        else
 936                idx = addr >> (6 + sck_way);
 937        idx = idx % ch_way;
 938
 939        /*
 940         * FIXME: Shouldn't we use CHN_IDX_OFFSET() here, when ch_way == 3 ???
 941         */
 942        switch (idx) {
 943        case 0:
 944                base_ch = TAD_TGT0(reg);
 945                break;
 946        case 1:
 947                base_ch = TAD_TGT1(reg);
 948                break;
 949        case 2:
 950                base_ch = TAD_TGT2(reg);
 951                break;
 952        case 3:
 953                base_ch = TAD_TGT3(reg);
 954                break;
 955        default:
 956                sprintf(msg, "Can't discover the TAD target");
 957                return -EINVAL;
 958        }
 959        *channel_mask = 1 << base_ch;
 960
 961        if (pvt->is_mirrored) {
 962                *channel_mask |= 1 << ((base_ch + 2) % 4);
 963                switch(ch_way) {
 964                case 2:
 965                case 4:
 966                        sck_xch = 1 << sck_way * (ch_way >> 1);
 967                        break;
 968                default:
 969                        sprintf(msg, "Invalid mirror set. Can't decode addr");
 970                        return -EINVAL;
 971                }
 972        } else
 973                sck_xch = (1 << sck_way) * ch_way;
 974
 975        if (pvt->is_lockstep)
 976                *channel_mask |= 1 << ((base_ch + 1) % 4);
 977
 978        offset = TAD_OFFSET(tad_offset);
 979
 980        edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n",
 981                 n_tads,
 982                 addr,
 983                 limit,
 984                 (u32)TAD_SOCK(reg),
 985                 ch_way,
 986                 offset,
 987                 idx,
 988                 base_ch,
 989                 *channel_mask);
 990
 991        /* Calculate channel address */
 992        /* Remove the TAD offset */
 993
 994        if (offset > addr) {
 995                sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!",
 996                        offset, addr);
 997                return -EINVAL;
 998        }
 999        addr -= offset;
1000        /* Store the low bits [0:6] of the addr */
1001        ch_addr = addr & 0x7f;
1002        /* Remove socket wayness and remove 6 bits */
1003        addr >>= 6;
1004        addr = div_u64(addr, sck_xch);
1005#if 0
1006        /* Divide by channel way */
1007        addr = addr / ch_way;
1008#endif
1009        /* Recover the last 6 bits */
1010        ch_addr |= addr << 6;
1011
1012        /*
1013         * Step 3) Decode rank
1014         */
1015        for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) {
1016                pci_read_config_dword(pvt->pci_tad[base_ch],
1017                                      rir_way_limit[n_rir],
1018                                      &reg);
1019
1020                if (!IS_RIR_VALID(reg))
1021                        continue;
1022
1023                limit = RIR_LIMIT(reg);
1024                mb = div_u64_rem(limit >> 20, 1000, &kb);
1025                edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n",
1026                         n_rir,
1027                         mb, kb,
1028                         limit,
1029                         1 << RIR_WAY(reg));
1030                if  (ch_addr <= limit)
1031                        break;
1032        }
1033        if (n_rir == MAX_RIR_RANGES) {
1034                sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx",
1035                        ch_addr);
1036                return -EINVAL;
1037        }
1038        rir_way = RIR_WAY(reg);
1039        if (pvt->is_close_pg)
1040                idx = (ch_addr >> 6);
1041        else
1042                idx = (ch_addr >> 13);  /* FIXME: Datasheet says to shift by 15 */
1043        idx %= 1 << rir_way;
1044
1045        pci_read_config_dword(pvt->pci_tad[base_ch],
1046                              rir_offset[n_rir][idx],
1047                              &reg);
1048        *rank = RIR_RNK_TGT(reg);
1049
1050        edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n",
1051                 n_rir,
1052                 ch_addr,
1053                 limit,
1054                 rir_way,
1055                 idx);
1056
1057        return 0;
1058}
1059
1060/****************************************************************************
1061        Device initialization routines: put/get, init/exit
1062 ****************************************************************************/
1063
1064/*
1065 *      sbridge_put_all_devices 'put' all the devices that we have
1066 *                              reserved via 'get'
1067 */
1068static void sbridge_put_devices(struct sbridge_dev *sbridge_dev)
1069{
1070        int i;
1071
1072        edac_dbg(0, "\n");
1073        for (i = 0; i < sbridge_dev->n_devs; i++) {
1074                struct pci_dev *pdev = sbridge_dev->pdev[i];
1075                if (!pdev)
1076                        continue;
1077                edac_dbg(0, "Removing dev %02x:%02x.%d\n",
1078                         pdev->bus->number,
1079                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1080                pci_dev_put(pdev);
1081        }
1082}
1083
1084static void sbridge_put_all_devices(void)
1085{
1086        struct sbridge_dev *sbridge_dev, *tmp;
1087
1088        list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) {
1089                sbridge_put_devices(sbridge_dev);
1090                free_sbridge_dev(sbridge_dev);
1091        }
1092}
1093
1094/*
1095 *      sbridge_get_all_devices Find and perform 'get' operation on the MCH's
1096 *                      device/functions we want to reference for this driver
1097 *
1098 *                      Need to 'get' device 16 func 1 and func 2
1099 */
1100static int sbridge_get_onedevice(struct pci_dev **prev,
1101                                 u8 *num_mc,
1102                                 const struct pci_id_table *table,
1103                                 const unsigned devno)
1104{
1105        struct sbridge_dev *sbridge_dev;
1106        const struct pci_id_descr *dev_descr = &table->descr[devno];
1107
1108        struct pci_dev *pdev = NULL;
1109        u8 bus = 0;
1110
1111        sbridge_printk(KERN_INFO,
1112                "Seeking for: dev %02x.%d PCI ID %04x:%04x\n",
1113                dev_descr->dev, dev_descr->func,
1114                PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1115
1116        pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1117                              dev_descr->dev_id, *prev);
1118
1119        if (!pdev) {
1120                if (*prev) {
1121                        *prev = pdev;
1122                        return 0;
1123                }
1124
1125                if (dev_descr->optional)
1126                        return 0;
1127
1128                if (devno == 0)
1129                        return -ENODEV;
1130
1131                sbridge_printk(KERN_INFO,
1132                        "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1133                        dev_descr->dev, dev_descr->func,
1134                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1135
1136                /* End of list, leave */
1137                return -ENODEV;
1138        }
1139        bus = pdev->bus->number;
1140
1141        sbridge_dev = get_sbridge_dev(bus);
1142        if (!sbridge_dev) {
1143                sbridge_dev = alloc_sbridge_dev(bus, table);
1144                if (!sbridge_dev) {
1145                        pci_dev_put(pdev);
1146                        return -ENOMEM;
1147                }
1148                (*num_mc)++;
1149        }
1150
1151        if (sbridge_dev->pdev[devno]) {
1152                sbridge_printk(KERN_ERR,
1153                        "Duplicated device for "
1154                        "dev %02x:%d.%d PCI ID %04x:%04x\n",
1155                        bus, dev_descr->dev, dev_descr->func,
1156                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1157                pci_dev_put(pdev);
1158                return -ENODEV;
1159        }
1160
1161        sbridge_dev->pdev[devno] = pdev;
1162
1163        /* Sanity check */
1164        if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1165                        PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1166                sbridge_printk(KERN_ERR,
1167                        "Device PCI ID %04x:%04x "
1168                        "has dev %02x:%d.%d instead of dev %02x:%02x.%d\n",
1169                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1170                        bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1171                        bus, dev_descr->dev, dev_descr->func);
1172                return -ENODEV;
1173        }
1174
1175        /* Be sure that the device is enabled */
1176        if (unlikely(pci_enable_device(pdev) < 0)) {
1177                sbridge_printk(KERN_ERR,
1178                        "Couldn't enable "
1179                        "dev %02x:%d.%d PCI ID %04x:%04x\n",
1180                        bus, dev_descr->dev, dev_descr->func,
1181                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1182                return -ENODEV;
1183        }
1184
1185        edac_dbg(0, "Detected dev %02x:%d.%d PCI ID %04x:%04x\n",
1186                 bus, dev_descr->dev, dev_descr->func,
1187                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1188
1189        /*
1190         * As stated on drivers/pci/search.c, the reference count for
1191         * @from is always decremented if it is not %NULL. So, as we need
1192         * to get all devices up to null, we need to do a get for the device
1193         */
1194        pci_dev_get(pdev);
1195
1196        *prev = pdev;
1197
1198        return 0;
1199}
1200
1201static int sbridge_get_all_devices(u8 *num_mc)
1202{
1203        int i, rc;
1204        struct pci_dev *pdev = NULL;
1205        const struct pci_id_table *table = pci_dev_descr_sbridge_table;
1206
1207        while (table && table->descr) {
1208                for (i = 0; i < table->n_devs; i++) {
1209                        pdev = NULL;
1210                        do {
1211                                rc = sbridge_get_onedevice(&pdev, num_mc,
1212                                                           table, i);
1213                                if (rc < 0) {
1214                                        if (i == 0) {
1215                                                i = table->n_devs;
1216                                                break;
1217                                        }
1218                                        sbridge_put_all_devices();
1219                                        return -ENODEV;
1220                                }
1221                        } while (pdev);
1222                }
1223                table++;
1224        }
1225
1226        return 0;
1227}
1228
1229static int mci_bind_devs(struct mem_ctl_info *mci,
1230                         struct sbridge_dev *sbridge_dev)
1231{
1232        struct sbridge_pvt *pvt = mci->pvt_info;
1233        struct pci_dev *pdev;
1234        int i, func, slot;
1235
1236        for (i = 0; i < sbridge_dev->n_devs; i++) {
1237                pdev = sbridge_dev->pdev[i];
1238                if (!pdev)
1239                        continue;
1240                slot = PCI_SLOT(pdev->devfn);
1241                func = PCI_FUNC(pdev->devfn);
1242                switch (slot) {
1243                case 12:
1244                        switch (func) {
1245                        case 6:
1246                                pvt->pci_sad0 = pdev;
1247                                break;
1248                        case 7:
1249                                pvt->pci_sad1 = pdev;
1250                                break;
1251                        default:
1252                                goto error;
1253                        }
1254                        break;
1255                case 13:
1256                        switch (func) {
1257                        case 6:
1258                                pvt->pci_br = pdev;
1259                                break;
1260                        default:
1261                                goto error;
1262                        }
1263                        break;
1264                case 14:
1265                        switch (func) {
1266                        case 0:
1267                                pvt->pci_ha0 = pdev;
1268                                break;
1269                        default:
1270                                goto error;
1271                        }
1272                        break;
1273                case 15:
1274                        switch (func) {
1275                        case 0:
1276                                pvt->pci_ta = pdev;
1277                                break;
1278                        case 1:
1279                                pvt->pci_ras = pdev;
1280                                break;
1281                        case 2:
1282                        case 3:
1283                        case 4:
1284                        case 5:
1285                                pvt->pci_tad[func - 2] = pdev;
1286                                break;
1287                        default:
1288                                goto error;
1289                        }
1290                        break;
1291                case 17:
1292                        switch (func) {
1293                        case 0:
1294                                pvt->pci_ddrio = pdev;
1295                                break;
1296                        default:
1297                                goto error;
1298                        }
1299                        break;
1300                default:
1301                        goto error;
1302                }
1303
1304                edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
1305                         sbridge_dev->bus,
1306                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1307                         pdev);
1308        }
1309
1310        /* Check if everything were registered */
1311        if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha0 ||
1312            !pvt-> pci_tad || !pvt->pci_ras  || !pvt->pci_ta)
1313                goto enodev;
1314
1315        for (i = 0; i < NUM_CHANNELS; i++) {
1316                if (!pvt->pci_tad[i])
1317                        goto enodev;
1318        }
1319        return 0;
1320
1321enodev:
1322        sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
1323        return -ENODEV;
1324
1325error:
1326        sbridge_printk(KERN_ERR, "Device %d, function %d "
1327                      "is out of the expected range\n",
1328                      slot, func);
1329        return -EINVAL;
1330}
1331
1332/****************************************************************************
1333                        Error check routines
1334 ****************************************************************************/
1335
1336/*
1337 * While Sandy Bridge has error count registers, SMI BIOS read values from
1338 * and resets the counters. So, they are not reliable for the OS to read
1339 * from them. So, we have no option but to just trust on whatever MCE is
1340 * telling us about the errors.
1341 */
1342static void sbridge_mce_output_error(struct mem_ctl_info *mci,
1343                                    const struct mce *m)
1344{
1345        struct mem_ctl_info *new_mci;
1346        struct sbridge_pvt *pvt = mci->pvt_info;
1347        enum hw_event_mc_err_type tp_event;
1348        char *type, *optype, msg[256];
1349        bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
1350        bool overflow = GET_BITFIELD(m->status, 62, 62);
1351        bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
1352        bool recoverable = GET_BITFIELD(m->status, 56, 56);
1353        u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
1354        u32 mscod = GET_BITFIELD(m->status, 16, 31);
1355        u32 errcode = GET_BITFIELD(m->status, 0, 15);
1356        u32 channel = GET_BITFIELD(m->status, 0, 3);
1357        u32 optypenum = GET_BITFIELD(m->status, 4, 6);
1358        long channel_mask, first_channel;
1359        u8  rank, socket;
1360        int rc, dimm;
1361        char *area_type = NULL;
1362
1363        if (uncorrected_error) {
1364                if (ripv) {
1365                        type = "FATAL";
1366                        tp_event = HW_EVENT_ERR_FATAL;
1367                } else {
1368                        type = "NON_FATAL";
1369                        tp_event = HW_EVENT_ERR_UNCORRECTED;
1370                }
1371        } else {
1372                type = "CORRECTED";
1373                tp_event = HW_EVENT_ERR_CORRECTED;
1374        }
1375
1376        /*
1377         * According with Table 15-9 of the Intel Architecture spec vol 3A,
1378         * memory errors should fit in this mask:
1379         *      000f 0000 1mmm cccc (binary)
1380         * where:
1381         *      f = Correction Report Filtering Bit. If 1, subsequent errors
1382         *          won't be shown
1383         *      mmm = error type
1384         *      cccc = channel
1385         * If the mask doesn't match, report an error to the parsing logic
1386         */
1387        if (! ((errcode & 0xef80) == 0x80)) {
1388                optype = "Can't parse: it is not a mem";
1389        } else {
1390                switch (optypenum) {
1391                case 0:
1392                        optype = "generic undef request error";
1393                        break;
1394                case 1:
1395                        optype = "memory read error";
1396                        break;
1397                case 2:
1398                        optype = "memory write error";
1399                        break;
1400                case 3:
1401                        optype = "addr/cmd error";
1402                        break;
1403                case 4:
1404                        optype = "memory scrubbing error";
1405                        break;
1406                default:
1407                        optype = "reserved";
1408                        break;
1409                }
1410        }
1411
1412        rc = get_memory_error_data(mci, m->addr, &socket,
1413                                   &channel_mask, &rank, &area_type, msg);
1414        if (rc < 0)
1415                goto err_parsing;
1416        new_mci = get_mci_for_node_id(socket);
1417        if (!new_mci) {
1418                strcpy(msg, "Error: socket got corrupted!");
1419                goto err_parsing;
1420        }
1421        mci = new_mci;
1422        pvt = mci->pvt_info;
1423
1424        first_channel = find_first_bit(&channel_mask, NUM_CHANNELS);
1425
1426        if (rank < 4)
1427                dimm = 0;
1428        else if (rank < 8)
1429                dimm = 1;
1430        else
1431                dimm = 2;
1432
1433
1434        /*
1435         * FIXME: On some memory configurations (mirror, lockstep), the
1436         * Memory Controller can't point the error to a single DIMM. The
1437         * EDAC core should be handling the channel mask, in order to point
1438         * to the group of dimm's where the error may be happening.
1439         */
1440        snprintf(msg, sizeof(msg),
1441                 "%s%s area:%s err_code:%04x:%04x socket:%d channel_mask:%ld rank:%d",
1442                 overflow ? " OVERFLOW" : "",
1443                 (uncorrected_error && recoverable) ? " recoverable" : "",
1444                 area_type,
1445                 mscod, errcode,
1446                 socket,
1447                 channel_mask,
1448                 rank);
1449
1450        edac_dbg(0, "%s\n", msg);
1451
1452        /* FIXME: need support for channel mask */
1453
1454        /* Call the helper to output message */
1455        edac_mc_handle_error(tp_event, mci, core_err_cnt,
1456                             m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
1457                             channel, dimm, -1,
1458                             optype, msg);
1459        return;
1460err_parsing:
1461        edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0,
1462                             -1, -1, -1,
1463                             msg, "");
1464
1465}
1466
1467/*
1468 *      sbridge_check_error     Retrieve and process errors reported by the
1469 *                              hardware. Called by the Core module.
1470 */
1471static void sbridge_check_error(struct mem_ctl_info *mci)
1472{
1473        struct sbridge_pvt *pvt = mci->pvt_info;
1474        int i;
1475        unsigned count = 0;
1476        struct mce *m;
1477
1478        /*
1479         * MCE first step: Copy all mce errors into a temporary buffer
1480         * We use a double buffering here, to reduce the risk of
1481         * loosing an error.
1482         */
1483        smp_rmb();
1484        count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1485                % MCE_LOG_LEN;
1486        if (!count)
1487                return;
1488
1489        m = pvt->mce_outentry;
1490        if (pvt->mce_in + count > MCE_LOG_LEN) {
1491                unsigned l = MCE_LOG_LEN - pvt->mce_in;
1492
1493                memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1494                smp_wmb();
1495                pvt->mce_in = 0;
1496                count -= l;
1497                m += l;
1498        }
1499        memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1500        smp_wmb();
1501        pvt->mce_in += count;
1502
1503        smp_rmb();
1504        if (pvt->mce_overrun) {
1505                sbridge_printk(KERN_ERR, "Lost %d memory errors\n",
1506                              pvt->mce_overrun);
1507                smp_wmb();
1508                pvt->mce_overrun = 0;
1509        }
1510
1511        /*
1512         * MCE second step: parse errors and display
1513         */
1514        for (i = 0; i < count; i++)
1515                sbridge_mce_output_error(mci, &pvt->mce_outentry[i]);
1516}
1517
1518/*
1519 * sbridge_mce_check_error      Replicates mcelog routine to get errors
1520 *                              This routine simply queues mcelog errors, and
1521 *                              return. The error itself should be handled later
1522 *                              by sbridge_check_error.
1523 * WARNING: As this routine should be called at NMI time, extra care should
1524 * be taken to avoid deadlocks, and to be as fast as possible.
1525 */
1526static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val,
1527                                   void *data)
1528{
1529        struct mce *mce = (struct mce *)data;
1530        struct mem_ctl_info *mci;
1531        struct sbridge_pvt *pvt;
1532
1533        mci = get_mci_for_node_id(mce->socketid);
1534        if (!mci)
1535                return NOTIFY_BAD;
1536        pvt = mci->pvt_info;
1537
1538        /*
1539         * Just let mcelog handle it if the error is
1540         * outside the memory controller. A memory error
1541         * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0.
1542         * bit 12 has an special meaning.
1543         */
1544        if ((mce->status & 0xefff) >> 7 != 1)
1545                return NOTIFY_DONE;
1546
1547        printk("sbridge: HANDLING MCE MEMORY ERROR\n");
1548
1549        printk("CPU %d: Machine Check Exception: %Lx Bank %d: %016Lx\n",
1550               mce->extcpu, mce->mcgstatus, mce->bank, mce->status);
1551        printk("TSC %llx ", mce->tsc);
1552        printk("ADDR %llx ", mce->addr);
1553        printk("MISC %llx ", mce->misc);
1554
1555        printk("PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
1556                mce->cpuvendor, mce->cpuid, mce->time,
1557                mce->socketid, mce->apicid);
1558
1559        /* Only handle if it is the right mc controller */
1560        if (cpu_data(mce->cpu).phys_proc_id != pvt->sbridge_dev->mc)
1561                return NOTIFY_DONE;
1562
1563        smp_rmb();
1564        if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1565                smp_wmb();
1566                pvt->mce_overrun++;
1567                return NOTIFY_DONE;
1568        }
1569
1570        /* Copy memory error at the ringbuffer */
1571        memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1572        smp_wmb();
1573        pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1574
1575        /* Handle fatal errors immediately */
1576        if (mce->mcgstatus & 1)
1577                sbridge_check_error(mci);
1578
1579        /* Advice mcelog that the error were handled */
1580        return NOTIFY_STOP;
1581}
1582
1583static struct notifier_block sbridge_mce_dec = {
1584        .notifier_call      = sbridge_mce_check_error,
1585};
1586
1587/****************************************************************************
1588                        EDAC register/unregister logic
1589 ****************************************************************************/
1590
1591static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev)
1592{
1593        struct mem_ctl_info *mci = sbridge_dev->mci;
1594        struct sbridge_pvt *pvt;
1595
1596        if (unlikely(!mci || !mci->pvt_info)) {
1597                edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev);
1598
1599                sbridge_printk(KERN_ERR, "Couldn't find mci handler\n");
1600                return;
1601        }
1602
1603        pvt = mci->pvt_info;
1604
1605        edac_dbg(0, "MC: mci = %p, dev = %p\n",
1606                 mci, &sbridge_dev->pdev[0]->dev);
1607
1608        /* Remove MC sysfs nodes */
1609        edac_mc_del_mc(mci->pdev);
1610
1611        edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
1612        kfree(mci->ctl_name);
1613        edac_mc_free(mci);
1614        sbridge_dev->mci = NULL;
1615}
1616
1617static int sbridge_register_mci(struct sbridge_dev *sbridge_dev)
1618{
1619        struct mem_ctl_info *mci;
1620        struct edac_mc_layer layers[2];
1621        struct sbridge_pvt *pvt;
1622        int rc;
1623
1624        /* Check the number of active and not disabled channels */
1625        rc = check_if_ecc_is_active(sbridge_dev->bus);
1626        if (unlikely(rc < 0))
1627                return rc;
1628
1629        /* allocate a new MC control structure */
1630        layers[0].type = EDAC_MC_LAYER_CHANNEL;
1631        layers[0].size = NUM_CHANNELS;
1632        layers[0].is_virt_csrow = false;
1633        layers[1].type = EDAC_MC_LAYER_SLOT;
1634        layers[1].size = MAX_DIMMS;
1635        layers[1].is_virt_csrow = true;
1636        mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers,
1637                            sizeof(*pvt));
1638
1639        if (unlikely(!mci))
1640                return -ENOMEM;
1641
1642        edac_dbg(0, "MC: mci = %p, dev = %p\n",
1643                 mci, &sbridge_dev->pdev[0]->dev);
1644
1645        pvt = mci->pvt_info;
1646        memset(pvt, 0, sizeof(*pvt));
1647
1648        /* Associate sbridge_dev and mci for future usage */
1649        pvt->sbridge_dev = sbridge_dev;
1650        sbridge_dev->mci = mci;
1651
1652        mci->mtype_cap = MEM_FLAG_DDR3;
1653        mci->edac_ctl_cap = EDAC_FLAG_NONE;
1654        mci->edac_cap = EDAC_FLAG_NONE;
1655        mci->mod_name = "sbridge_edac.c";
1656        mci->mod_ver = SBRIDGE_REVISION;
1657        mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge Socket#%d", mci->mc_idx);
1658        mci->dev_name = pci_name(sbridge_dev->pdev[0]);
1659        mci->ctl_page_to_phys = NULL;
1660
1661        /* Set the function pointer to an actual operation function */
1662        mci->edac_check = sbridge_check_error;
1663
1664        /* Store pci devices at mci for faster access */
1665        rc = mci_bind_devs(mci, sbridge_dev);
1666        if (unlikely(rc < 0))
1667                goto fail0;
1668
1669        /* Get dimm basic config and the memory layout */
1670        get_dimm_config(mci);
1671        get_memory_layout(mci);
1672
1673        /* record ptr to the generic device */
1674        mci->pdev = &sbridge_dev->pdev[0]->dev;
1675
1676        /* add this new MC control structure to EDAC's list of MCs */
1677        if (unlikely(edac_mc_add_mc(mci))) {
1678                edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
1679                rc = -EINVAL;
1680                goto fail0;
1681        }
1682
1683        return 0;
1684
1685fail0:
1686        kfree(mci->ctl_name);
1687        edac_mc_free(mci);
1688        sbridge_dev->mci = NULL;
1689        return rc;
1690}
1691
1692/*
1693 *      sbridge_probe   Probe for ONE instance of device to see if it is
1694 *                      present.
1695 *      return:
1696 *              0 for FOUND a device
1697 *              < 0 for error code
1698 */
1699
1700static int sbridge_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1701{
1702        int rc;
1703        u8 mc, num_mc = 0;
1704        struct sbridge_dev *sbridge_dev;
1705
1706        /* get the pci devices we want to reserve for our use */
1707        mutex_lock(&sbridge_edac_lock);
1708
1709        /*
1710         * All memory controllers are allocated at the first pass.
1711         */
1712        if (unlikely(probed >= 1)) {
1713                mutex_unlock(&sbridge_edac_lock);
1714                return -ENODEV;
1715        }
1716        probed++;
1717
1718        rc = sbridge_get_all_devices(&num_mc);
1719        if (unlikely(rc < 0))
1720                goto fail0;
1721        mc = 0;
1722
1723        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
1724                edac_dbg(0, "Registering MC#%d (%d of %d)\n",
1725                         mc, mc + 1, num_mc);
1726                sbridge_dev->mc = mc++;
1727                rc = sbridge_register_mci(sbridge_dev);
1728                if (unlikely(rc < 0))
1729                        goto fail1;
1730        }
1731
1732        sbridge_printk(KERN_INFO, "Driver loaded.\n");
1733
1734        mutex_unlock(&sbridge_edac_lock);
1735        return 0;
1736
1737fail1:
1738        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
1739                sbridge_unregister_mci(sbridge_dev);
1740
1741        sbridge_put_all_devices();
1742fail0:
1743        mutex_unlock(&sbridge_edac_lock);
1744        return rc;
1745}
1746
1747/*
1748 *      sbridge_remove  destructor for one instance of device
1749 *
1750 */
1751static void sbridge_remove(struct pci_dev *pdev)
1752{
1753        struct sbridge_dev *sbridge_dev;
1754
1755        edac_dbg(0, "\n");
1756
1757        /*
1758         * we have a trouble here: pdev value for removal will be wrong, since
1759         * it will point to the X58 register used to detect that the machine
1760         * is a Nehalem or upper design. However, due to the way several PCI
1761         * devices are grouped together to provide MC functionality, we need
1762         * to use a different method for releasing the devices
1763         */
1764
1765        mutex_lock(&sbridge_edac_lock);
1766
1767        if (unlikely(!probed)) {
1768                mutex_unlock(&sbridge_edac_lock);
1769                return;
1770        }
1771
1772        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
1773                sbridge_unregister_mci(sbridge_dev);
1774
1775        /* Release PCI resources */
1776        sbridge_put_all_devices();
1777
1778        probed--;
1779
1780        mutex_unlock(&sbridge_edac_lock);
1781}
1782
1783MODULE_DEVICE_TABLE(pci, sbridge_pci_tbl);
1784
1785/*
1786 *      sbridge_driver  pci_driver structure for this module
1787 *
1788 */
1789static struct pci_driver sbridge_driver = {
1790        .name     = "sbridge_edac",
1791        .probe    = sbridge_probe,
1792        .remove   = sbridge_remove,
1793        .id_table = sbridge_pci_tbl,
1794};
1795
1796/*
1797 *      sbridge_init            Module entry function
1798 *                      Try to initialize this module for its devices
1799 */
1800static int __init sbridge_init(void)
1801{
1802        int pci_rc;
1803
1804        edac_dbg(2, "\n");
1805
1806        /* Ensure that the OPSTATE is set correctly for POLL or NMI */
1807        opstate_init();
1808
1809        pci_rc = pci_register_driver(&sbridge_driver);
1810
1811        if (pci_rc >= 0) {
1812                mce_register_decode_chain(&sbridge_mce_dec);
1813                return 0;
1814        }
1815
1816        sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n",
1817                      pci_rc);
1818
1819        return pci_rc;
1820}
1821
1822/*
1823 *      sbridge_exit()  Module exit function
1824 *                      Unregister the driver
1825 */
1826static void __exit sbridge_exit(void)
1827{
1828        edac_dbg(2, "\n");
1829        pci_unregister_driver(&sbridge_driver);
1830        mce_unregister_decode_chain(&sbridge_mce_dec);
1831}
1832
1833module_init(sbridge_init);
1834module_exit(sbridge_exit);
1835
1836module_param(edac_op_state, int, 0444);
1837MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
1838
1839MODULE_LICENSE("GPL");
1840MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
1841MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
1842MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge memory controllers - "
1843                   SBRIDGE_REVISION);
1844
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.