linux/drivers/edac/sb_edac.c
<<
>>
Prefs
   1/* Intel Sandy Bridge -EN/-EP/-EX Memory Controller kernel module
   2 *
   3 * This driver supports the memory controllers found on the Intel
   4 * processor family Sandy Bridge.
   5 *
   6 * This file may be distributed under the terms of the
   7 * GNU General Public License version 2 only.
   8 *
   9 * Copyright (c) 2011 by:
  10 *       Mauro Carvalho Chehab <mchehab@redhat.com>
  11 */
  12
  13#include <linux/module.h>
  14#include <linux/init.h>
  15#include <linux/pci.h>
  16#include <linux/pci_ids.h>
  17#include <linux/slab.h>
  18#include <linux/delay.h>
  19#include <linux/edac.h>
  20#include <linux/mmzone.h>
  21#include <linux/smp.h>
  22#include <linux/bitmap.h>
  23#include <linux/math64.h>
  24#include <asm/processor.h>
  25#include <asm/mce.h>
  26
  27#include "edac_core.h"
  28
  29/* Static vars */
  30static LIST_HEAD(sbridge_edac_list);
  31static DEFINE_MUTEX(sbridge_edac_lock);
  32static int probed;
  33
  34/*
  35 * Alter this version for the module when modifications are made
  36 */
  37#define SBRIDGE_REVISION    " Ver: 1.0.0 "
  38#define EDAC_MOD_STR      "sbridge_edac"
  39
  40/*
  41 * Debug macros
  42 */
  43#define sbridge_printk(level, fmt, arg...)                      \
  44        edac_printk(level, "sbridge", fmt, ##arg)
  45
  46#define sbridge_mc_printk(mci, level, fmt, arg...)              \
  47        edac_mc_chipset_printk(mci, level, "sbridge", fmt, ##arg)
  48
  49/*
  50 * Get a bit field at register value <v>, from bit <lo> to bit <hi>
  51 */
  52#define GET_BITFIELD(v, lo, hi) \
  53        (((v) & ((1ULL << ((hi) - (lo) + 1)) - 1) << (lo)) >> (lo))
  54
  55/*
  56 * sbridge Memory Controller Registers
  57 */
  58
  59/*
  60 * FIXME: For now, let's order by device function, as it makes
  61 * easier for driver's development process. This table should be
  62 * moved to pci_id.h when submitted upstream
  63 */
  64#define PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0        0x3cf4  /* 12.6 */
  65#define PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1        0x3cf6  /* 12.7 */
  66#define PCI_DEVICE_ID_INTEL_SBRIDGE_BR          0x3cf5  /* 13.6 */
  67#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0     0x3ca0  /* 14.0 */
  68#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA      0x3ca8  /* 15.0 */
  69#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS     0x3c71  /* 15.1 */
  70#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0    0x3caa  /* 15.2 */
  71#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1    0x3cab  /* 15.3 */
  72#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2    0x3cac  /* 15.4 */
  73#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3    0x3cad  /* 15.5 */
  74#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO   0x3cb8  /* 17.0 */
  75
  76        /*
  77         * Currently, unused, but will be needed in the future
  78         * implementations, as they hold the error counters
  79         */
  80#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR0    0x3c72  /* 16.2 */
  81#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR1    0x3c73  /* 16.3 */
  82#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR2    0x3c76  /* 16.6 */
  83#define PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_ERR3    0x3c77  /* 16.7 */
  84
  85/* Devices 12 Function 6, Offsets 0x80 to 0xcc */
  86static const u32 dram_rule[] = {
  87        0x80, 0x88, 0x90, 0x98, 0xa0,
  88        0xa8, 0xb0, 0xb8, 0xc0, 0xc8,
  89};
  90#define MAX_SAD         ARRAY_SIZE(dram_rule)
  91
  92#define SAD_LIMIT(reg)          ((GET_BITFIELD(reg, 6, 25) << 26) | 0x3ffffff)
  93#define DRAM_ATTR(reg)          GET_BITFIELD(reg, 2,  3)
  94#define INTERLEAVE_MODE(reg)    GET_BITFIELD(reg, 1,  1)
  95#define DRAM_RULE_ENABLE(reg)   GET_BITFIELD(reg, 0,  0)
  96
  97static char *get_dram_attr(u32 reg)
  98{
  99        switch(DRAM_ATTR(reg)) {
 100                case 0:
 101                        return "DRAM";
 102                case 1:
 103                        return "MMCFG";
 104                case 2:
 105                        return "NXM";
 106                default:
 107                        return "unknown";
 108        }
 109}
 110
 111static const u32 interleave_list[] = {
 112        0x84, 0x8c, 0x94, 0x9c, 0xa4,
 113        0xac, 0xb4, 0xbc, 0xc4, 0xcc,
 114};
 115#define MAX_INTERLEAVE  ARRAY_SIZE(interleave_list)
 116
 117#define SAD_PKG0(reg)           GET_BITFIELD(reg, 0, 2)
 118#define SAD_PKG1(reg)           GET_BITFIELD(reg, 3, 5)
 119#define SAD_PKG2(reg)           GET_BITFIELD(reg, 8, 10)
 120#define SAD_PKG3(reg)           GET_BITFIELD(reg, 11, 13)
 121#define SAD_PKG4(reg)           GET_BITFIELD(reg, 16, 18)
 122#define SAD_PKG5(reg)           GET_BITFIELD(reg, 19, 21)
 123#define SAD_PKG6(reg)           GET_BITFIELD(reg, 24, 26)
 124#define SAD_PKG7(reg)           GET_BITFIELD(reg, 27, 29)
 125
 126static inline int sad_pkg(u32 reg, int interleave)
 127{
 128        switch (interleave) {
 129        case 0:
 130                return SAD_PKG0(reg);
 131        case 1:
 132                return SAD_PKG1(reg);
 133        case 2:
 134                return SAD_PKG2(reg);
 135        case 3:
 136                return SAD_PKG3(reg);
 137        case 4:
 138                return SAD_PKG4(reg);
 139        case 5:
 140                return SAD_PKG5(reg);
 141        case 6:
 142                return SAD_PKG6(reg);
 143        case 7:
 144                return SAD_PKG7(reg);
 145        default:
 146                return -EINVAL;
 147        }
 148}
 149
 150/* Devices 12 Function 7 */
 151
 152#define TOLM            0x80
 153#define TOHM            0x84
 154
 155#define GET_TOLM(reg)           ((GET_BITFIELD(reg, 0,  3) << 28) | 0x3ffffff)
 156#define GET_TOHM(reg)           ((GET_BITFIELD(reg, 0, 20) << 25) | 0x3ffffff)
 157
 158/* Device 13 Function 6 */
 159
 160#define SAD_TARGET      0xf0
 161
 162#define SOURCE_ID(reg)          GET_BITFIELD(reg, 9, 11)
 163
 164#define SAD_CONTROL     0xf4
 165
 166#define NODE_ID(reg)            GET_BITFIELD(reg, 0, 2)
 167
 168/* Device 14 function 0 */
 169
 170static const u32 tad_dram_rule[] = {
 171        0x40, 0x44, 0x48, 0x4c,
 172        0x50, 0x54, 0x58, 0x5c,
 173        0x60, 0x64, 0x68, 0x6c,
 174};
 175#define MAX_TAD ARRAY_SIZE(tad_dram_rule)
 176
 177#define TAD_LIMIT(reg)          ((GET_BITFIELD(reg, 12, 31) << 26) | 0x3ffffff)
 178#define TAD_SOCK(reg)           GET_BITFIELD(reg, 10, 11)
 179#define TAD_CH(reg)             GET_BITFIELD(reg,  8,  9)
 180#define TAD_TGT3(reg)           GET_BITFIELD(reg,  6,  7)
 181#define TAD_TGT2(reg)           GET_BITFIELD(reg,  4,  5)
 182#define TAD_TGT1(reg)           GET_BITFIELD(reg,  2,  3)
 183#define TAD_TGT0(reg)           GET_BITFIELD(reg,  0,  1)
 184
 185/* Device 15, function 0 */
 186
 187#define MCMTR                   0x7c
 188
 189#define IS_ECC_ENABLED(mcmtr)           GET_BITFIELD(mcmtr, 2, 2)
 190#define IS_LOCKSTEP_ENABLED(mcmtr)      GET_BITFIELD(mcmtr, 1, 1)
 191#define IS_CLOSE_PG(mcmtr)              GET_BITFIELD(mcmtr, 0, 0)
 192
 193/* Device 15, function 1 */
 194
 195#define RASENABLES              0xac
 196#define IS_MIRROR_ENABLED(reg)          GET_BITFIELD(reg, 0, 0)
 197
 198/* Device 15, functions 2-5 */
 199
 200static const int mtr_regs[] = {
 201        0x80, 0x84, 0x88,
 202};
 203
 204#define RANK_DISABLE(mtr)               GET_BITFIELD(mtr, 16, 19)
 205#define IS_DIMM_PRESENT(mtr)            GET_BITFIELD(mtr, 14, 14)
 206#define RANK_CNT_BITS(mtr)              GET_BITFIELD(mtr, 12, 13)
 207#define RANK_WIDTH_BITS(mtr)            GET_BITFIELD(mtr, 2, 4)
 208#define COL_WIDTH_BITS(mtr)             GET_BITFIELD(mtr, 0, 1)
 209
 210static const u32 tad_ch_nilv_offset[] = {
 211        0x90, 0x94, 0x98, 0x9c,
 212        0xa0, 0xa4, 0xa8, 0xac,
 213        0xb0, 0xb4, 0xb8, 0xbc,
 214};
 215#define CHN_IDX_OFFSET(reg)             GET_BITFIELD(reg, 28, 29)
 216#define TAD_OFFSET(reg)                 (GET_BITFIELD(reg,  6, 25) << 26)
 217
 218static const u32 rir_way_limit[] = {
 219        0x108, 0x10c, 0x110, 0x114, 0x118,
 220};
 221#define MAX_RIR_RANGES ARRAY_SIZE(rir_way_limit)
 222
 223#define IS_RIR_VALID(reg)       GET_BITFIELD(reg, 31, 31)
 224#define RIR_WAY(reg)            GET_BITFIELD(reg, 28, 29)
 225#define RIR_LIMIT(reg)          ((GET_BITFIELD(reg,  1, 10) << 29)| 0x1fffffff)
 226
 227#define MAX_RIR_WAY     8
 228
 229static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = {
 230        { 0x120, 0x124, 0x128, 0x12c, 0x130, 0x134, 0x138, 0x13c },
 231        { 0x140, 0x144, 0x148, 0x14c, 0x150, 0x154, 0x158, 0x15c },
 232        { 0x160, 0x164, 0x168, 0x16c, 0x170, 0x174, 0x178, 0x17c },
 233        { 0x180, 0x184, 0x188, 0x18c, 0x190, 0x194, 0x198, 0x19c },
 234        { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc },
 235};
 236
 237#define RIR_RNK_TGT(reg)                GET_BITFIELD(reg, 16, 19)
 238#define RIR_OFFSET(reg)         GET_BITFIELD(reg,  2, 14)
 239
 240/* Device 16, functions 2-7 */
 241
 242/*
 243 * FIXME: Implement the error count reads directly
 244 */
 245
 246static const u32 correrrcnt[] = {
 247        0x104, 0x108, 0x10c, 0x110,
 248};
 249
 250#define RANK_ODD_OV(reg)                GET_BITFIELD(reg, 31, 31)
 251#define RANK_ODD_ERR_CNT(reg)           GET_BITFIELD(reg, 16, 30)
 252#define RANK_EVEN_OV(reg)               GET_BITFIELD(reg, 15, 15)
 253#define RANK_EVEN_ERR_CNT(reg)          GET_BITFIELD(reg,  0, 14)
 254
 255static const u32 correrrthrsld[] = {
 256        0x11c, 0x120, 0x124, 0x128,
 257};
 258
 259#define RANK_ODD_ERR_THRSLD(reg)        GET_BITFIELD(reg, 16, 30)
 260#define RANK_EVEN_ERR_THRSLD(reg)       GET_BITFIELD(reg,  0, 14)
 261
 262
 263/* Device 17, function 0 */
 264
 265#define RANK_CFG_A              0x0328
 266
 267#define IS_RDIMM_ENABLED(reg)           GET_BITFIELD(reg, 11, 11)
 268
 269/*
 270 * sbridge structs
 271 */
 272
 273#define NUM_CHANNELS    4
 274#define MAX_DIMMS       3               /* Max DIMMS per channel */
 275
 276struct sbridge_info {
 277        u32     mcmtr;
 278};
 279
 280struct sbridge_channel {
 281        u32             ranks;
 282        u32             dimms;
 283};
 284
 285struct pci_id_descr {
 286        int                     dev;
 287        int                     func;
 288        int                     dev_id;
 289        int                     optional;
 290};
 291
 292struct pci_id_table {
 293        const struct pci_id_descr       *descr;
 294        int                             n_devs;
 295};
 296
 297struct sbridge_dev {
 298        struct list_head        list;
 299        u8                      bus, mc;
 300        u8                      node_id, source_id;
 301        struct pci_dev          **pdev;
 302        int                     n_devs;
 303        struct mem_ctl_info     *mci;
 304};
 305
 306struct sbridge_pvt {
 307        struct pci_dev          *pci_ta, *pci_ddrio, *pci_ras;
 308        struct pci_dev          *pci_sad0, *pci_sad1, *pci_ha0;
 309        struct pci_dev          *pci_br;
 310        struct pci_dev          *pci_tad[NUM_CHANNELS];
 311
 312        struct sbridge_dev      *sbridge_dev;
 313
 314        struct sbridge_info     info;
 315        struct sbridge_channel  channel[NUM_CHANNELS];
 316
 317        /* Memory type detection */
 318        bool                    is_mirrored, is_lockstep, is_close_pg;
 319
 320        /* Fifo double buffers */
 321        struct mce              mce_entry[MCE_LOG_LEN];
 322        struct mce              mce_outentry[MCE_LOG_LEN];
 323
 324        /* Fifo in/out counters */
 325        unsigned                mce_in, mce_out;
 326
 327        /* Count indicator to show errors not got */
 328        unsigned                mce_overrun;
 329
 330        /* Memory description */
 331        u64                     tolm, tohm;
 332};
 333
 334#define PCI_DESCR(device, function, device_id)  \
 335        .dev = (device),                        \
 336        .func = (function),                     \
 337        .dev_id = (device_id)
 338
 339static const struct pci_id_descr pci_dev_descr_sbridge[] = {
 340                /* Processor Home Agent */
 341        { PCI_DESCR(14, 0, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_HA0)         },
 342
 343                /* Memory controller */
 344        { PCI_DESCR(15, 0, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA)          },
 345        { PCI_DESCR(15, 1, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_RAS)         },
 346        { PCI_DESCR(15, 2, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD0)        },
 347        { PCI_DESCR(15, 3, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD1)        },
 348        { PCI_DESCR(15, 4, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD2)        },
 349        { PCI_DESCR(15, 5, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TAD3)        },
 350        { PCI_DESCR(17, 0, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_DDRIO)       },
 351
 352                /* System Address Decoder */
 353        { PCI_DESCR(12, 6, PCI_DEVICE_ID_INTEL_SBRIDGE_SAD0)            },
 354        { PCI_DESCR(12, 7, PCI_DEVICE_ID_INTEL_SBRIDGE_SAD1)            },
 355
 356                /* Broadcast Registers */
 357        { PCI_DESCR(13, 6, PCI_DEVICE_ID_INTEL_SBRIDGE_BR)              },
 358};
 359
 360#define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 361static const struct pci_id_table pci_dev_descr_sbridge_table[] = {
 362        PCI_ID_TABLE_ENTRY(pci_dev_descr_sbridge),
 363        {0,}                    /* 0 terminated list. */
 364};
 365
 366/*
 367 *      pci_device_id   table for which devices we are looking for
 368 */
 369static DEFINE_PCI_DEVICE_TABLE(sbridge_pci_tbl) = {
 370        {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SBRIDGE_IMC_TA)},
 371        {0,}                    /* 0 terminated list. */
 372};
 373
 374
 375/****************************************************************************
 376                        Ancillary status routines
 377 ****************************************************************************/
 378
 379static inline int numrank(u32 mtr)
 380{
 381        int ranks = (1 << RANK_CNT_BITS(mtr));
 382
 383        if (ranks > 4) {
 384                edac_dbg(0, "Invalid number of ranks: %d (max = 4) raw value = %x (%04x)\n",
 385                         ranks, (unsigned int)RANK_CNT_BITS(mtr), mtr);
 386                return -EINVAL;
 387        }
 388
 389        return ranks;
 390}
 391
 392static inline int numrow(u32 mtr)
 393{
 394        int rows = (RANK_WIDTH_BITS(mtr) + 12);
 395
 396        if (rows < 13 || rows > 18) {
 397                edac_dbg(0, "Invalid number of rows: %d (should be between 14 and 17) raw value = %x (%04x)\n",
 398                         rows, (unsigned int)RANK_WIDTH_BITS(mtr), mtr);
 399                return -EINVAL;
 400        }
 401
 402        return 1 << rows;
 403}
 404
 405static inline int numcol(u32 mtr)
 406{
 407        int cols = (COL_WIDTH_BITS(mtr) + 10);
 408
 409        if (cols > 12) {
 410                edac_dbg(0, "Invalid number of cols: %d (max = 4) raw value = %x (%04x)\n",
 411                         cols, (unsigned int)COL_WIDTH_BITS(mtr), mtr);
 412                return -EINVAL;
 413        }
 414
 415        return 1 << cols;
 416}
 417
 418static struct sbridge_dev *get_sbridge_dev(u8 bus)
 419{
 420        struct sbridge_dev *sbridge_dev;
 421
 422        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
 423                if (sbridge_dev->bus == bus)
 424                        return sbridge_dev;
 425        }
 426
 427        return NULL;
 428}
 429
 430static struct sbridge_dev *alloc_sbridge_dev(u8 bus,
 431                                           const struct pci_id_table *table)
 432{
 433        struct sbridge_dev *sbridge_dev;
 434
 435        sbridge_dev = kzalloc(sizeof(*sbridge_dev), GFP_KERNEL);
 436        if (!sbridge_dev)
 437                return NULL;
 438
 439        sbridge_dev->pdev = kzalloc(sizeof(*sbridge_dev->pdev) * table->n_devs,
 440                                   GFP_KERNEL);
 441        if (!sbridge_dev->pdev) {
 442                kfree(sbridge_dev);
 443                return NULL;
 444        }
 445
 446        sbridge_dev->bus = bus;
 447        sbridge_dev->n_devs = table->n_devs;
 448        list_add_tail(&sbridge_dev->list, &sbridge_edac_list);
 449
 450        return sbridge_dev;
 451}
 452
 453static void free_sbridge_dev(struct sbridge_dev *sbridge_dev)
 454{
 455        list_del(&sbridge_dev->list);
 456        kfree(sbridge_dev->pdev);
 457        kfree(sbridge_dev);
 458}
 459
 460/****************************************************************************
 461                        Memory check routines
 462 ****************************************************************************/
 463static struct pci_dev *get_pdev_slot_func(u8 bus, unsigned slot,
 464                                          unsigned func)
 465{
 466        struct sbridge_dev *sbridge_dev = get_sbridge_dev(bus);
 467        int i;
 468
 469        if (!sbridge_dev)
 470                return NULL;
 471
 472        for (i = 0; i < sbridge_dev->n_devs; i++) {
 473                if (!sbridge_dev->pdev[i])
 474                        continue;
 475
 476                if (PCI_SLOT(sbridge_dev->pdev[i]->devfn) == slot &&
 477                    PCI_FUNC(sbridge_dev->pdev[i]->devfn) == func) {
 478                        edac_dbg(1, "Associated %02x.%02x.%d with %p\n",
 479                                 bus, slot, func, sbridge_dev->pdev[i]);
 480                        return sbridge_dev->pdev[i];
 481                }
 482        }
 483
 484        return NULL;
 485}
 486
 487/**
 488 * check_if_ecc_is_active() - Checks if ECC is active
 489 * bus:         Device bus
 490 */
 491static int check_if_ecc_is_active(const u8 bus)
 492{
 493        struct pci_dev *pdev = NULL;
 494        u32 mcmtr;
 495
 496        pdev = get_pdev_slot_func(bus, 15, 0);
 497        if (!pdev) {
 498                sbridge_printk(KERN_ERR, "Couldn't find PCI device "
 499                                        "%2x.%02d.%d!!!\n",
 500                                        bus, 15, 0);
 501                return -ENODEV;
 502        }
 503
 504        pci_read_config_dword(pdev, MCMTR, &mcmtr);
 505        if (!IS_ECC_ENABLED(mcmtr)) {
 506                sbridge_printk(KERN_ERR, "ECC is disabled. Aborting\n");
 507                return -ENODEV;
 508        }
 509        return 0;
 510}
 511
 512static int get_dimm_config(struct mem_ctl_info *mci)
 513{
 514        struct sbridge_pvt *pvt = mci->pvt_info;
 515        struct dimm_info *dimm;
 516        unsigned i, j, banks, ranks, rows, cols, npages;
 517        u64 size;
 518        u32 reg;
 519        enum edac_type mode;
 520        enum mem_type mtype;
 521
 522        pci_read_config_dword(pvt->pci_br, SAD_TARGET, &reg);
 523        pvt->sbridge_dev->source_id = SOURCE_ID(reg);
 524
 525        pci_read_config_dword(pvt->pci_br, SAD_CONTROL, &reg);
 526        pvt->sbridge_dev->node_id = NODE_ID(reg);
 527        edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n",
 528                 pvt->sbridge_dev->mc,
 529                 pvt->sbridge_dev->node_id,
 530                 pvt->sbridge_dev->source_id);
 531
 532        pci_read_config_dword(pvt->pci_ras, RASENABLES, &reg);
 533        if (IS_MIRROR_ENABLED(reg)) {
 534                edac_dbg(0, "Memory mirror is enabled\n");
 535                pvt->is_mirrored = true;
 536        } else {
 537                edac_dbg(0, "Memory mirror is disabled\n");
 538                pvt->is_mirrored = false;
 539        }
 540
 541        pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr);
 542        if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) {
 543                edac_dbg(0, "Lockstep is enabled\n");
 544                mode = EDAC_S8ECD8ED;
 545                pvt->is_lockstep = true;
 546        } else {
 547                edac_dbg(0, "Lockstep is disabled\n");
 548                mode = EDAC_S4ECD4ED;
 549                pvt->is_lockstep = false;
 550        }
 551        if (IS_CLOSE_PG(pvt->info.mcmtr)) {
 552                edac_dbg(0, "address map is on closed page mode\n");
 553                pvt->is_close_pg = true;
 554        } else {
 555                edac_dbg(0, "address map is on open page mode\n");
 556                pvt->is_close_pg = false;
 557        }
 558
 559        pci_read_config_dword(pvt->pci_ddrio, RANK_CFG_A, &reg);
 560        if (IS_RDIMM_ENABLED(reg)) {
 561                /* FIXME: Can also be LRDIMM */
 562                edac_dbg(0, "Memory is registered\n");
 563                mtype = MEM_RDDR3;
 564        } else {
 565                edac_dbg(0, "Memory is unregistered\n");
 566                mtype = MEM_DDR3;
 567        }
 568
 569        /* On all supported DDR3 DIMM types, there are 8 banks available */
 570        banks = 8;
 571
 572        for (i = 0; i < NUM_CHANNELS; i++) {
 573                u32 mtr;
 574
 575                for (j = 0; j < ARRAY_SIZE(mtr_regs); j++) {
 576                        dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers,
 577                                       i, j, 0);
 578                        pci_read_config_dword(pvt->pci_tad[i],
 579                                              mtr_regs[j], &mtr);
 580                        edac_dbg(4, "Channel #%d  MTR%d = %x\n", i, j, mtr);
 581                        if (IS_DIMM_PRESENT(mtr)) {
 582                                pvt->channel[i].dimms++;
 583
 584                                ranks = numrank(mtr);
 585                                rows = numrow(mtr);
 586                                cols = numcol(mtr);
 587
 588                                /* DDR3 has 8 I/O banks */
 589                                size = ((u64)rows * cols * banks * ranks) >> (20 - 3);
 590                                npages = MiB_TO_PAGES(size);
 591
 592                                edac_dbg(0, "mc#%d: channel %d, dimm %d, %Ld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n",
 593                                         pvt->sbridge_dev->mc, i, j,
 594                                         size, npages,
 595                                         banks, ranks, rows, cols);
 596
 597                                dimm->nr_pages = npages;
 598                                dimm->grain = 32;
 599                                dimm->dtype = (banks == 8) ? DEV_X8 : DEV_X4;
 600                                dimm->mtype = mtype;
 601                                dimm->edac_mode = mode;
 602                                snprintf(dimm->label, sizeof(dimm->label),
 603                                         "CPU_SrcID#%u_Channel#%u_DIMM#%u",
 604                                         pvt->sbridge_dev->source_id, i, j);
 605                        }
 606                }
 607        }
 608
 609        return 0;
 610}
 611
 612static void get_memory_layout(const struct mem_ctl_info *mci)
 613{
 614        struct sbridge_pvt *pvt = mci->pvt_info;
 615        int i, j, k, n_sads, n_tads, sad_interl;
 616        u32 reg;
 617        u64 limit, prv = 0;
 618        u64 tmp_mb;
 619        u32 mb, kb;
 620        u32 rir_way;
 621
 622        /*
 623         * Step 1) Get TOLM/TOHM ranges
 624         */
 625
 626        /* Address range is 32:28 */
 627        pci_read_config_dword(pvt->pci_sad1, TOLM,
 628                              &reg);
 629        pvt->tolm = GET_TOLM(reg);
 630        tmp_mb = (1 + pvt->tolm) >> 20;
 631
 632        mb = div_u64_rem(tmp_mb, 1000, &kb);
 633        edac_dbg(0, "TOLM: %u.%03u GB (0x%016Lx)\n", mb, kb, (u64)pvt->tolm);
 634
 635        /* Address range is already 45:25 */
 636        pci_read_config_dword(pvt->pci_sad1, TOHM,
 637                              &reg);
 638        pvt->tohm = GET_TOHM(reg);
 639        tmp_mb = (1 + pvt->tohm) >> 20;
 640
 641        mb = div_u64_rem(tmp_mb, 1000, &kb);
 642        edac_dbg(0, "TOHM: %u.%03u GB (0x%016Lx)\n", mb, kb, (u64)pvt->tohm);
 643
 644        /*
 645         * Step 2) Get SAD range and SAD Interleave list
 646         * TAD registers contain the interleave wayness. However, it
 647         * seems simpler to just discover it indirectly, with the
 648         * algorithm bellow.
 649         */
 650        prv = 0;
 651        for (n_sads = 0; n_sads < MAX_SAD; n_sads++) {
 652                /* SAD_LIMIT Address range is 45:26 */
 653                pci_read_config_dword(pvt->pci_sad0, dram_rule[n_sads],
 654                                      &reg);
 655                limit = SAD_LIMIT(reg);
 656
 657                if (!DRAM_RULE_ENABLE(reg))
 658                        continue;
 659
 660                if (limit <= prv)
 661                        break;
 662
 663                tmp_mb = (limit + 1) >> 20;
 664                mb = div_u64_rem(tmp_mb, 1000, &kb);
 665                edac_dbg(0, "SAD#%d %s up to %u.%03u GB (0x%016Lx) Interleave: %s reg=0x%08x\n",
 666                         n_sads,
 667                         get_dram_attr(reg),
 668                         mb, kb,
 669                         ((u64)tmp_mb) << 20L,
 670                         INTERLEAVE_MODE(reg) ? "8:6" : "[8:6]XOR[18:16]",
 671                         reg);
 672                prv = limit;
 673
 674                pci_read_config_dword(pvt->pci_sad0, interleave_list[n_sads],
 675                                      &reg);
 676                sad_interl = sad_pkg(reg, 0);
 677                for (j = 0; j < 8; j++) {
 678                        if (j > 0 && sad_interl == sad_pkg(reg, j))
 679                                break;
 680
 681                        edac_dbg(0, "SAD#%d, interleave #%d: %d\n",
 682                                 n_sads, j, sad_pkg(reg, j));
 683                }
 684        }
 685
 686        /*
 687         * Step 3) Get TAD range
 688         */
 689        prv = 0;
 690        for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
 691                pci_read_config_dword(pvt->pci_ha0, tad_dram_rule[n_tads],
 692                                      &reg);
 693                limit = TAD_LIMIT(reg);
 694                if (limit <= prv)
 695                        break;
 696                tmp_mb = (limit + 1) >> 20;
 697
 698                mb = div_u64_rem(tmp_mb, 1000, &kb);
 699                edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n",
 700                         n_tads, mb, kb,
 701                         ((u64)tmp_mb) << 20L,
 702                         (u32)TAD_SOCK(reg),
 703                         (u32)TAD_CH(reg),
 704                         (u32)TAD_TGT0(reg),
 705                         (u32)TAD_TGT1(reg),
 706                         (u32)TAD_TGT2(reg),
 707                         (u32)TAD_TGT3(reg),
 708                         reg);
 709                prv = limit;
 710        }
 711
 712        /*
 713         * Step 4) Get TAD offsets, per each channel
 714         */
 715        for (i = 0; i < NUM_CHANNELS; i++) {
 716                if (!pvt->channel[i].dimms)
 717                        continue;
 718                for (j = 0; j < n_tads; j++) {
 719                        pci_read_config_dword(pvt->pci_tad[i],
 720                                              tad_ch_nilv_offset[j],
 721                                              &reg);
 722                        tmp_mb = TAD_OFFSET(reg) >> 20;
 723                        mb = div_u64_rem(tmp_mb, 1000, &kb);
 724                        edac_dbg(0, "TAD CH#%d, offset #%d: %u.%03u GB (0x%016Lx), reg=0x%08x\n",
 725                                 i, j,
 726                                 mb, kb,
 727                                 ((u64)tmp_mb) << 20L,
 728                                 reg);
 729                }
 730        }
 731
 732        /*
 733         * Step 6) Get RIR Wayness/Limit, per each channel
 734         */
 735        for (i = 0; i < NUM_CHANNELS; i++) {
 736                if (!pvt->channel[i].dimms)
 737                        continue;
 738                for (j = 0; j < MAX_RIR_RANGES; j++) {
 739                        pci_read_config_dword(pvt->pci_tad[i],
 740                                              rir_way_limit[j],
 741                                              &reg);
 742
 743                        if (!IS_RIR_VALID(reg))
 744                                continue;
 745
 746                        tmp_mb = RIR_LIMIT(reg) >> 20;
 747                        rir_way = 1 << RIR_WAY(reg);
 748                        mb = div_u64_rem(tmp_mb, 1000, &kb);
 749                        edac_dbg(0, "CH#%d RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d, reg=0x%08x\n",
 750                                 i, j,
 751                                 mb, kb,
 752                                 ((u64)tmp_mb) << 20L,
 753                                 rir_way,
 754                                 reg);
 755
 756                        for (k = 0; k < rir_way; k++) {
 757                                pci_read_config_dword(pvt->pci_tad[i],
 758                                                      rir_offset[j][k],
 759                                                      &reg);
 760                                tmp_mb = RIR_OFFSET(reg) << 6;
 761
 762                                mb = div_u64_rem(tmp_mb, 1000, &kb);
 763                                edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n",
 764                                         i, j, k,
 765                                         mb, kb,
 766                                         ((u64)tmp_mb) << 20L,
 767                                         (u32)RIR_RNK_TGT(reg),
 768                                         reg);
 769                        }
 770                }
 771        }
 772}
 773
 774struct mem_ctl_info *get_mci_for_node_id(u8 node_id)
 775{
 776        struct sbridge_dev *sbridge_dev;
 777
 778        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
 779                if (sbridge_dev->node_id == node_id)
 780                        return sbridge_dev->mci;
 781        }
 782        return NULL;
 783}
 784
 785static int get_memory_error_data(struct mem_ctl_info *mci,
 786                                 u64 addr,
 787                                 u8 *socket,
 788                                 long *channel_mask,
 789                                 u8 *rank,
 790                                 char **area_type, char *msg)
 791{
 792        struct mem_ctl_info     *new_mci;
 793        struct sbridge_pvt *pvt = mci->pvt_info;
 794        int                     n_rir, n_sads, n_tads, sad_way, sck_xch;
 795        int                     sad_interl, idx, base_ch;
 796        int                     interleave_mode;
 797        unsigned                sad_interleave[MAX_INTERLEAVE];
 798        u32                     reg;
 799        u8                      ch_way,sck_way;
 800        u32                     tad_offset;
 801        u32                     rir_way;
 802        u32                     mb, kb;
 803        u64                     ch_addr, offset, limit, prv = 0;
 804
 805
 806        /*
 807         * Step 0) Check if the address is at special memory ranges
 808         * The check bellow is probably enough to fill all cases where
 809         * the error is not inside a memory, except for the legacy
 810         * range (e. g. VGA addresses). It is unlikely, however, that the
 811         * memory controller would generate an error on that range.
 812         */
 813        if ((addr > (u64) pvt->tolm) && (addr < (1LL << 32))) {
 814                sprintf(msg, "Error at TOLM area, on addr 0x%08Lx", addr);
 815                return -EINVAL;
 816        }
 817        if (addr >= (u64)pvt->tohm) {
 818                sprintf(msg, "Error at MMIOH area, on addr 0x%016Lx", addr);
 819                return -EINVAL;
 820        }
 821
 822        /*
 823         * Step 1) Get socket
 824         */
 825        for (n_sads = 0; n_sads < MAX_SAD; n_sads++) {
 826                pci_read_config_dword(pvt->pci_sad0, dram_rule[n_sads],
 827                                      &reg);
 828
 829                if (!DRAM_RULE_ENABLE(reg))
 830                        continue;
 831
 832                limit = SAD_LIMIT(reg);
 833                if (limit <= prv) {
 834                        sprintf(msg, "Can't discover the memory socket");
 835                        return -EINVAL;
 836                }
 837                if  (addr <= limit)
 838                        break;
 839                prv = limit;
 840        }
 841        if (n_sads == MAX_SAD) {
 842                sprintf(msg, "Can't discover the memory socket");
 843                return -EINVAL;
 844        }
 845        *area_type = get_dram_attr(reg);
 846        interleave_mode = INTERLEAVE_MODE(reg);
 847
 848        pci_read_config_dword(pvt->pci_sad0, interleave_list[n_sads],
 849                              &reg);
 850        sad_interl = sad_pkg(reg, 0);
 851        for (sad_way = 0; sad_way < 8; sad_way++) {
 852                if (sad_way > 0 && sad_interl == sad_pkg(reg, sad_way))
 853                        break;
 854                sad_interleave[sad_way] = sad_pkg(reg, sad_way);
 855                edac_dbg(0, "SAD interleave #%d: %d\n",
 856                         sad_way, sad_interleave[sad_way]);
 857        }
 858        edac_dbg(0, "mc#%d: Error detected on SAD#%d: address 0x%016Lx < 0x%016Lx, Interleave [%d:6]%s\n",
 859                 pvt->sbridge_dev->mc,
 860                 n_sads,
 861                 addr,
 862                 limit,
 863                 sad_way + 7,
 864                 interleave_mode ? "" : "XOR[18:16]");
 865        if (interleave_mode)
 866                idx = ((addr >> 6) ^ (addr >> 16)) & 7;
 867        else
 868                idx = (addr >> 6) & 7;
 869        switch (sad_way) {
 870        case 1:
 871                idx = 0;
 872                break;
 873        case 2:
 874                idx = idx & 1;
 875                break;
 876        case 4:
 877                idx = idx & 3;
 878                break;
 879        case 8:
 880                break;
 881        default:
 882                sprintf(msg, "Can't discover socket interleave");
 883                return -EINVAL;
 884        }
 885        *socket = sad_interleave[idx];
 886        edac_dbg(0, "SAD interleave index: %d (wayness %d) = CPU socket %d\n",
 887                 idx, sad_way, *socket);
 888
 889        /*
 890         * Move to the proper node structure, in order to access the
 891         * right PCI registers
 892         */
 893        new_mci = get_mci_for_node_id(*socket);
 894        if (!new_mci) {
 895                sprintf(msg, "Struct for socket #%u wasn't initialized",
 896                        *socket);
 897                return -EINVAL;
 898        }
 899        mci = new_mci;
 900        pvt = mci->pvt_info;
 901
 902        /*
 903         * Step 2) Get memory channel
 904         */
 905        prv = 0;
 906        for (n_tads = 0; n_tads < MAX_TAD; n_tads++) {
 907                pci_read_config_dword(pvt->pci_ha0, tad_dram_rule[n_tads],
 908                                      &reg);
 909                limit = TAD_LIMIT(reg);
 910                if (limit <= prv) {
 911                        sprintf(msg, "Can't discover the memory channel");
 912                        return -EINVAL;
 913                }
 914                if  (addr <= limit)
 915                        break;
 916                prv = limit;
 917        }
 918        ch_way = TAD_CH(reg) + 1;
 919        sck_way = TAD_SOCK(reg) + 1;
 920        /*
 921         * FIXME: Is it right to always use channel 0 for offsets?
 922         */
 923        pci_read_config_dword(pvt->pci_tad[0],
 924                                tad_ch_nilv_offset[n_tads],
 925                                &tad_offset);
 926
 927        if (ch_way == 3)
 928                idx = addr >> 6;
 929        else
 930                idx = addr >> (6 + sck_way);
 931        idx = idx % ch_way;
 932
 933        /*
 934         * FIXME: Shouldn't we use CHN_IDX_OFFSET() here, when ch_way == 3 ???
 935         */
 936        switch (idx) {
 937        case 0:
 938                base_ch = TAD_TGT0(reg);
 939                break;
 940        case 1:
 941                base_ch = TAD_TGT1(reg);
 942                break;
 943        case 2:
 944                base_ch = TAD_TGT2(reg);
 945                break;
 946        case 3:
 947                base_ch = TAD_TGT3(reg);
 948                break;
 949        default:
 950                sprintf(msg, "Can't discover the TAD target");
 951                return -EINVAL;
 952        }
 953        *channel_mask = 1 << base_ch;
 954
 955        if (pvt->is_mirrored) {
 956                *channel_mask |= 1 << ((base_ch + 2) % 4);
 957                switch(ch_way) {
 958                case 2:
 959                case 4:
 960                        sck_xch = 1 << sck_way * (ch_way >> 1);
 961                        break;
 962                default:
 963                        sprintf(msg, "Invalid mirror set. Can't decode addr");
 964                        return -EINVAL;
 965                }
 966        } else
 967                sck_xch = (1 << sck_way) * ch_way;
 968
 969        if (pvt->is_lockstep)
 970                *channel_mask |= 1 << ((base_ch + 1) % 4);
 971
 972        offset = TAD_OFFSET(tad_offset);
 973
 974        edac_dbg(0, "TAD#%d: address 0x%016Lx < 0x%016Lx, socket interleave %d, channel interleave %d (offset 0x%08Lx), index %d, base ch: %d, ch mask: 0x%02lx\n",
 975                 n_tads,
 976                 addr,
 977                 limit,
 978                 (u32)TAD_SOCK(reg),
 979                 ch_way,
 980                 offset,
 981                 idx,
 982                 base_ch,
 983                 *channel_mask);
 984
 985        /* Calculate channel address */
 986        /* Remove the TAD offset */
 987
 988        if (offset > addr) {
 989                sprintf(msg, "Can't calculate ch addr: TAD offset 0x%08Lx is too high for addr 0x%08Lx!",
 990                        offset, addr);
 991                return -EINVAL;
 992        }
 993        addr -= offset;
 994        /* Store the low bits [0:6] of the addr */
 995        ch_addr = addr & 0x7f;
 996        /* Remove socket wayness and remove 6 bits */
 997        addr >>= 6;
 998        addr = div_u64(addr, sck_xch);
 999#if 0
1000        /* Divide by channel way */
1001        addr = addr / ch_way;
1002#endif
1003        /* Recover the last 6 bits */
1004        ch_addr |= addr << 6;
1005
1006        /*
1007         * Step 3) Decode rank
1008         */
1009        for (n_rir = 0; n_rir < MAX_RIR_RANGES; n_rir++) {
1010                pci_read_config_dword(pvt->pci_tad[base_ch],
1011                                      rir_way_limit[n_rir],
1012                                      &reg);
1013
1014                if (!IS_RIR_VALID(reg))
1015                        continue;
1016
1017                limit = RIR_LIMIT(reg);
1018                mb = div_u64_rem(limit >> 20, 1000, &kb);
1019                edac_dbg(0, "RIR#%d, limit: %u.%03u GB (0x%016Lx), way: %d\n",
1020                         n_rir,
1021                         mb, kb,
1022                         limit,
1023                         1 << RIR_WAY(reg));
1024                if  (ch_addr <= limit)
1025                        break;
1026        }
1027        if (n_rir == MAX_RIR_RANGES) {
1028                sprintf(msg, "Can't discover the memory rank for ch addr 0x%08Lx",
1029                        ch_addr);
1030                return -EINVAL;
1031        }
1032        rir_way = RIR_WAY(reg);
1033        if (pvt->is_close_pg)
1034                idx = (ch_addr >> 6);
1035        else
1036                idx = (ch_addr >> 13);  /* FIXME: Datasheet says to shift by 15 */
1037        idx %= 1 << rir_way;
1038
1039        pci_read_config_dword(pvt->pci_tad[base_ch],
1040                              rir_offset[n_rir][idx],
1041                              &reg);
1042        *rank = RIR_RNK_TGT(reg);
1043
1044        edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n",
1045                 n_rir,
1046                 ch_addr,
1047                 limit,
1048                 rir_way,
1049                 idx);
1050
1051        return 0;
1052}
1053
1054/****************************************************************************
1055        Device initialization routines: put/get, init/exit
1056 ****************************************************************************/
1057
1058/*
1059 *      sbridge_put_all_devices 'put' all the devices that we have
1060 *                              reserved via 'get'
1061 */
1062static void sbridge_put_devices(struct sbridge_dev *sbridge_dev)
1063{
1064        int i;
1065
1066        edac_dbg(0, "\n");
1067        for (i = 0; i < sbridge_dev->n_devs; i++) {
1068                struct pci_dev *pdev = sbridge_dev->pdev[i];
1069                if (!pdev)
1070                        continue;
1071                edac_dbg(0, "Removing dev %02x:%02x.%d\n",
1072                         pdev->bus->number,
1073                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1074                pci_dev_put(pdev);
1075        }
1076}
1077
1078static void sbridge_put_all_devices(void)
1079{
1080        struct sbridge_dev *sbridge_dev, *tmp;
1081
1082        list_for_each_entry_safe(sbridge_dev, tmp, &sbridge_edac_list, list) {
1083                sbridge_put_devices(sbridge_dev);
1084                free_sbridge_dev(sbridge_dev);
1085        }
1086}
1087
1088/*
1089 *      sbridge_get_all_devices Find and perform 'get' operation on the MCH's
1090 *                      device/functions we want to reference for this driver
1091 *
1092 *                      Need to 'get' device 16 func 1 and func 2
1093 */
1094static int sbridge_get_onedevice(struct pci_dev **prev,
1095                                 u8 *num_mc,
1096                                 const struct pci_id_table *table,
1097                                 const unsigned devno)
1098{
1099        struct sbridge_dev *sbridge_dev;
1100        const struct pci_id_descr *dev_descr = &table->descr[devno];
1101
1102        struct pci_dev *pdev = NULL;
1103        u8 bus = 0;
1104
1105        sbridge_printk(KERN_INFO,
1106                "Seeking for: dev %02x.%d PCI ID %04x:%04x\n",
1107                dev_descr->dev, dev_descr->func,
1108                PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1109
1110        pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1111                              dev_descr->dev_id, *prev);
1112
1113        if (!pdev) {
1114                if (*prev) {
1115                        *prev = pdev;
1116                        return 0;
1117                }
1118
1119                if (dev_descr->optional)
1120                        return 0;
1121
1122                if (devno == 0)
1123                        return -ENODEV;
1124
1125                sbridge_printk(KERN_INFO,
1126                        "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1127                        dev_descr->dev, dev_descr->func,
1128                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1129
1130                /* End of list, leave */
1131                return -ENODEV;
1132        }
1133        bus = pdev->bus->number;
1134
1135        sbridge_dev = get_sbridge_dev(bus);
1136        if (!sbridge_dev) {
1137                sbridge_dev = alloc_sbridge_dev(bus, table);
1138                if (!sbridge_dev) {
1139                        pci_dev_put(pdev);
1140                        return -ENOMEM;
1141                }
1142                (*num_mc)++;
1143        }
1144
1145        if (sbridge_dev->pdev[devno]) {
1146                sbridge_printk(KERN_ERR,
1147                        "Duplicated device for "
1148                        "dev %02x:%d.%d PCI ID %04x:%04x\n",
1149                        bus, dev_descr->dev, dev_descr->func,
1150                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1151                pci_dev_put(pdev);
1152                return -ENODEV;
1153        }
1154
1155        sbridge_dev->pdev[devno] = pdev;
1156
1157        /* Sanity check */
1158        if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1159                        PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1160                sbridge_printk(KERN_ERR,
1161                        "Device PCI ID %04x:%04x "
1162                        "has dev %02x:%d.%d instead of dev %02x:%02x.%d\n",
1163                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1164                        bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1165                        bus, dev_descr->dev, dev_descr->func);
1166                return -ENODEV;
1167        }
1168
1169        /* Be sure that the device is enabled */
1170        if (unlikely(pci_enable_device(pdev) < 0)) {
1171                sbridge_printk(KERN_ERR,
1172                        "Couldn't enable "
1173                        "dev %02x:%d.%d PCI ID %04x:%04x\n",
1174                        bus, dev_descr->dev, dev_descr->func,
1175                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1176                return -ENODEV;
1177        }
1178
1179        edac_dbg(0, "Detected dev %02x:%d.%d PCI ID %04x:%04x\n",
1180                 bus, dev_descr->dev, dev_descr->func,
1181                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1182
1183        /*
1184         * As stated on drivers/pci/search.c, the reference count for
1185         * @from is always decremented if it is not %NULL. So, as we need
1186         * to get all devices up to null, we need to do a get for the device
1187         */
1188        pci_dev_get(pdev);
1189
1190        *prev = pdev;
1191
1192        return 0;
1193}
1194
1195static int sbridge_get_all_devices(u8 *num_mc)
1196{
1197        int i, rc;
1198        struct pci_dev *pdev = NULL;
1199        const struct pci_id_table *table = pci_dev_descr_sbridge_table;
1200
1201        while (table && table->descr) {
1202                for (i = 0; i < table->n_devs; i++) {
1203                        pdev = NULL;
1204                        do {
1205                                rc = sbridge_get_onedevice(&pdev, num_mc,
1206                                                           table, i);
1207                                if (rc < 0) {
1208                                        if (i == 0) {
1209                                                i = table->n_devs;
1210                                                break;
1211                                        }
1212                                        sbridge_put_all_devices();
1213                                        return -ENODEV;
1214                                }
1215                        } while (pdev);
1216                }
1217                table++;
1218        }
1219
1220        return 0;
1221}
1222
1223static int mci_bind_devs(struct mem_ctl_info *mci,
1224                         struct sbridge_dev *sbridge_dev)
1225{
1226        struct sbridge_pvt *pvt = mci->pvt_info;
1227        struct pci_dev *pdev;
1228        int i, func, slot;
1229
1230        for (i = 0; i < sbridge_dev->n_devs; i++) {
1231                pdev = sbridge_dev->pdev[i];
1232                if (!pdev)
1233                        continue;
1234                slot = PCI_SLOT(pdev->devfn);
1235                func = PCI_FUNC(pdev->devfn);
1236                switch (slot) {
1237                case 12:
1238                        switch (func) {
1239                        case 6:
1240                                pvt->pci_sad0 = pdev;
1241                                break;
1242                        case 7:
1243                                pvt->pci_sad1 = pdev;
1244                                break;
1245                        default:
1246                                goto error;
1247                        }
1248                        break;
1249                case 13:
1250                        switch (func) {
1251                        case 6:
1252                                pvt->pci_br = pdev;
1253                                break;
1254                        default:
1255                                goto error;
1256                        }
1257                        break;
1258                case 14:
1259                        switch (func) {
1260                        case 0:
1261                                pvt->pci_ha0 = pdev;
1262                                break;
1263                        default:
1264                                goto error;
1265                        }
1266                        break;
1267                case 15:
1268                        switch (func) {
1269                        case 0:
1270                                pvt->pci_ta = pdev;
1271                                break;
1272                        case 1:
1273                                pvt->pci_ras = pdev;
1274                                break;
1275                        case 2:
1276                        case 3:
1277                        case 4:
1278                        case 5:
1279                                pvt->pci_tad[func - 2] = pdev;
1280                                break;
1281                        default:
1282                                goto error;
1283                        }
1284                        break;
1285                case 17:
1286                        switch (func) {
1287                        case 0:
1288                                pvt->pci_ddrio = pdev;
1289                                break;
1290                        default:
1291                                goto error;
1292                        }
1293                        break;
1294                default:
1295                        goto error;
1296                }
1297
1298                edac_dbg(0, "Associated PCI %02x.%02d.%d with dev = %p\n",
1299                         sbridge_dev->bus,
1300                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1301                         pdev);
1302        }
1303
1304        /* Check if everything were registered */
1305        if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha0 ||
1306            !pvt-> pci_tad || !pvt->pci_ras  || !pvt->pci_ta ||
1307            !pvt->pci_ddrio)
1308                goto enodev;
1309
1310        for (i = 0; i < NUM_CHANNELS; i++) {
1311                if (!pvt->pci_tad[i])
1312                        goto enodev;
1313        }
1314        return 0;
1315
1316enodev:
1317        sbridge_printk(KERN_ERR, "Some needed devices are missing\n");
1318        return -ENODEV;
1319
1320error:
1321        sbridge_printk(KERN_ERR, "Device %d, function %d "
1322                      "is out of the expected range\n",
1323                      slot, func);
1324        return -EINVAL;
1325}
1326
1327/****************************************************************************
1328                        Error check routines
1329 ****************************************************************************/
1330
1331/*
1332 * While Sandy Bridge has error count registers, SMI BIOS read values from
1333 * and resets the counters. So, they are not reliable for the OS to read
1334 * from them. So, we have no option but to just trust on whatever MCE is
1335 * telling us about the errors.
1336 */
1337static void sbridge_mce_output_error(struct mem_ctl_info *mci,
1338                                    const struct mce *m)
1339{
1340        struct mem_ctl_info *new_mci;
1341        struct sbridge_pvt *pvt = mci->pvt_info;
1342        enum hw_event_mc_err_type tp_event;
1343        char *type, *optype, msg[256];
1344        bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
1345        bool overflow = GET_BITFIELD(m->status, 62, 62);
1346        bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
1347        bool recoverable = GET_BITFIELD(m->status, 56, 56);
1348        u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
1349        u32 mscod = GET_BITFIELD(m->status, 16, 31);
1350        u32 errcode = GET_BITFIELD(m->status, 0, 15);
1351        u32 channel = GET_BITFIELD(m->status, 0, 3);
1352        u32 optypenum = GET_BITFIELD(m->status, 4, 6);
1353        long channel_mask, first_channel;
1354        u8  rank, socket;
1355        int rc, dimm;
1356        char *area_type = NULL;
1357
1358        if (uncorrected_error) {
1359                if (ripv) {
1360                        type = "FATAL";
1361                        tp_event = HW_EVENT_ERR_FATAL;
1362                } else {
1363                        type = "NON_FATAL";
1364                        tp_event = HW_EVENT_ERR_UNCORRECTED;
1365                }
1366        } else {
1367                type = "CORRECTED";
1368                tp_event = HW_EVENT_ERR_CORRECTED;
1369        }
1370
1371        /*
1372         * According with Table 15-9 of the Intel Architecture spec vol 3A,
1373         * memory errors should fit in this mask:
1374         *      000f 0000 1mmm cccc (binary)
1375         * where:
1376         *      f = Correction Report Filtering Bit. If 1, subsequent errors
1377         *          won't be shown
1378         *      mmm = error type
1379         *      cccc = channel
1380         * If the mask doesn't match, report an error to the parsing logic
1381         */
1382        if (! ((errcode & 0xef80) == 0x80)) {
1383                optype = "Can't parse: it is not a mem";
1384        } else {
1385                switch (optypenum) {
1386                case 0:
1387                        optype = "generic undef request error";
1388                        break;
1389                case 1:
1390                        optype = "memory read error";
1391                        break;
1392                case 2:
1393                        optype = "memory write error";
1394                        break;
1395                case 3:
1396                        optype = "addr/cmd error";
1397                        break;
1398                case 4:
1399                        optype = "memory scrubbing error";
1400                        break;
1401                default:
1402                        optype = "reserved";
1403                        break;
1404                }
1405        }
1406
1407        rc = get_memory_error_data(mci, m->addr, &socket,
1408                                   &channel_mask, &rank, &area_type, msg);
1409        if (rc < 0)
1410                goto err_parsing;
1411        new_mci = get_mci_for_node_id(socket);
1412        if (!new_mci) {
1413                strcpy(msg, "Error: socket got corrupted!");
1414                goto err_parsing;
1415        }
1416        mci = new_mci;
1417        pvt = mci->pvt_info;
1418
1419        first_channel = find_first_bit(&channel_mask, NUM_CHANNELS);
1420
1421        if (rank < 4)
1422                dimm = 0;
1423        else if (rank < 8)
1424                dimm = 1;
1425        else
1426                dimm = 2;
1427
1428
1429        /*
1430         * FIXME: On some memory configurations (mirror, lockstep), the
1431         * Memory Controller can't point the error to a single DIMM. The
1432         * EDAC core should be handling the channel mask, in order to point
1433         * to the group of dimm's where the error may be happening.
1434         */
1435        snprintf(msg, sizeof(msg),
1436                 "%s%s area:%s err_code:%04x:%04x socket:%d channel_mask:%ld rank:%d",
1437                 overflow ? " OVERFLOW" : "",
1438                 (uncorrected_error && recoverable) ? " recoverable" : "",
1439                 area_type,
1440                 mscod, errcode,
1441                 socket,
1442                 channel_mask,
1443                 rank);
1444
1445        edac_dbg(0, "%s\n", msg);
1446
1447        /* FIXME: need support for channel mask */
1448
1449        /* Call the helper to output message */
1450        edac_mc_handle_error(tp_event, mci, core_err_cnt,
1451                             m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
1452                             channel, dimm, -1,
1453                             optype, msg);
1454        return;
1455err_parsing:
1456        edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0,
1457                             -1, -1, -1,
1458                             msg, "");
1459
1460}
1461
1462/*
1463 *      sbridge_check_error     Retrieve and process errors reported by the
1464 *                              hardware. Called by the Core module.
1465 */
1466static void sbridge_check_error(struct mem_ctl_info *mci)
1467{
1468        struct sbridge_pvt *pvt = mci->pvt_info;
1469        int i;
1470        unsigned count = 0;
1471        struct mce *m;
1472
1473        /*
1474         * MCE first step: Copy all mce errors into a temporary buffer
1475         * We use a double buffering here, to reduce the risk of
1476         * loosing an error.
1477         */
1478        smp_rmb();
1479        count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1480                % MCE_LOG_LEN;
1481        if (!count)
1482                return;
1483
1484        m = pvt->mce_outentry;
1485        if (pvt->mce_in + count > MCE_LOG_LEN) {
1486                unsigned l = MCE_LOG_LEN - pvt->mce_in;
1487
1488                memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1489                smp_wmb();
1490                pvt->mce_in = 0;
1491                count -= l;
1492                m += l;
1493        }
1494        memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1495        smp_wmb();
1496        pvt->mce_in += count;
1497
1498        smp_rmb();
1499        if (pvt->mce_overrun) {
1500                sbridge_printk(KERN_ERR, "Lost %d memory errors\n",
1501                              pvt->mce_overrun);
1502                smp_wmb();
1503                pvt->mce_overrun = 0;
1504        }
1505
1506        /*
1507         * MCE second step: parse errors and display
1508         */
1509        for (i = 0; i < count; i++)
1510                sbridge_mce_output_error(mci, &pvt->mce_outentry[i]);
1511}
1512
1513/*
1514 * sbridge_mce_check_error      Replicates mcelog routine to get errors
1515 *                              This routine simply queues mcelog errors, and
1516 *                              return. The error itself should be handled later
1517 *                              by sbridge_check_error.
1518 * WARNING: As this routine should be called at NMI time, extra care should
1519 * be taken to avoid deadlocks, and to be as fast as possible.
1520 */
1521static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val,
1522                                   void *data)
1523{
1524        struct mce *mce = (struct mce *)data;
1525        struct mem_ctl_info *mci;
1526        struct sbridge_pvt *pvt;
1527
1528        mci = get_mci_for_node_id(mce->socketid);
1529        if (!mci)
1530                return NOTIFY_BAD;
1531        pvt = mci->pvt_info;
1532
1533        /*
1534         * Just let mcelog handle it if the error is
1535         * outside the memory controller. A memory error
1536         * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0.
1537         * bit 12 has an special meaning.
1538         */
1539        if ((mce->status & 0xefff) >> 7 != 1)
1540                return NOTIFY_DONE;
1541
1542        printk("sbridge: HANDLING MCE MEMORY ERROR\n");
1543
1544        printk("CPU %d: Machine Check Exception: %Lx Bank %d: %016Lx\n",
1545               mce->extcpu, mce->mcgstatus, mce->bank, mce->status);
1546        printk("TSC %llx ", mce->tsc);
1547        printk("ADDR %llx ", mce->addr);
1548        printk("MISC %llx ", mce->misc);
1549
1550        printk("PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
1551                mce->cpuvendor, mce->cpuid, mce->time,
1552                mce->socketid, mce->apicid);
1553
1554        /* Only handle if it is the right mc controller */
1555        if (cpu_data(mce->cpu).phys_proc_id != pvt->sbridge_dev->mc)
1556                return NOTIFY_DONE;
1557
1558        smp_rmb();
1559        if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1560                smp_wmb();
1561                pvt->mce_overrun++;
1562                return NOTIFY_DONE;
1563        }
1564
1565        /* Copy memory error at the ringbuffer */
1566        memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1567        smp_wmb();
1568        pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1569
1570        /* Handle fatal errors immediately */
1571        if (mce->mcgstatus & 1)
1572                sbridge_check_error(mci);
1573
1574        /* Advice mcelog that the error were handled */
1575        return NOTIFY_STOP;
1576}
1577
1578static struct notifier_block sbridge_mce_dec = {
1579        .notifier_call      = sbridge_mce_check_error,
1580};
1581
1582/****************************************************************************
1583                        EDAC register/unregister logic
1584 ****************************************************************************/
1585
1586static void sbridge_unregister_mci(struct sbridge_dev *sbridge_dev)
1587{
1588        struct mem_ctl_info *mci = sbridge_dev->mci;
1589        struct sbridge_pvt *pvt;
1590
1591        if (unlikely(!mci || !mci->pvt_info)) {
1592                edac_dbg(0, "MC: dev = %p\n", &sbridge_dev->pdev[0]->dev);
1593
1594                sbridge_printk(KERN_ERR, "Couldn't find mci handler\n");
1595                return;
1596        }
1597
1598        pvt = mci->pvt_info;
1599
1600        edac_dbg(0, "MC: mci = %p, dev = %p\n",
1601                 mci, &sbridge_dev->pdev[0]->dev);
1602
1603        /* Remove MC sysfs nodes */
1604        edac_mc_del_mc(mci->pdev);
1605
1606        edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
1607        kfree(mci->ctl_name);
1608        edac_mc_free(mci);
1609        sbridge_dev->mci = NULL;
1610}
1611
1612static int sbridge_register_mci(struct sbridge_dev *sbridge_dev)
1613{
1614        struct mem_ctl_info *mci;
1615        struct edac_mc_layer layers[2];
1616        struct sbridge_pvt *pvt;
1617        int rc;
1618
1619        /* Check the number of active and not disabled channels */
1620        rc = check_if_ecc_is_active(sbridge_dev->bus);
1621        if (unlikely(rc < 0))
1622                return rc;
1623
1624        /* allocate a new MC control structure */
1625        layers[0].type = EDAC_MC_LAYER_CHANNEL;
1626        layers[0].size = NUM_CHANNELS;
1627        layers[0].is_virt_csrow = false;
1628        layers[1].type = EDAC_MC_LAYER_SLOT;
1629        layers[1].size = MAX_DIMMS;
1630        layers[1].is_virt_csrow = true;
1631        mci = edac_mc_alloc(sbridge_dev->mc, ARRAY_SIZE(layers), layers,
1632                            sizeof(*pvt));
1633
1634        if (unlikely(!mci))
1635                return -ENOMEM;
1636
1637        edac_dbg(0, "MC: mci = %p, dev = %p\n",
1638                 mci, &sbridge_dev->pdev[0]->dev);
1639
1640        pvt = mci->pvt_info;
1641        memset(pvt, 0, sizeof(*pvt));
1642
1643        /* Associate sbridge_dev and mci for future usage */
1644        pvt->sbridge_dev = sbridge_dev;
1645        sbridge_dev->mci = mci;
1646
1647        mci->mtype_cap = MEM_FLAG_DDR3;
1648        mci->edac_ctl_cap = EDAC_FLAG_NONE;
1649        mci->edac_cap = EDAC_FLAG_NONE;
1650        mci->mod_name = "sbridge_edac.c";
1651        mci->mod_ver = SBRIDGE_REVISION;
1652        mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge Socket#%d", mci->mc_idx);
1653        mci->dev_name = pci_name(sbridge_dev->pdev[0]);
1654        mci->ctl_page_to_phys = NULL;
1655
1656        /* Set the function pointer to an actual operation function */
1657        mci->edac_check = sbridge_check_error;
1658
1659        /* Store pci devices at mci for faster access */
1660        rc = mci_bind_devs(mci, sbridge_dev);
1661        if (unlikely(rc < 0))
1662                goto fail0;
1663
1664        /* Get dimm basic config and the memory layout */
1665        get_dimm_config(mci);
1666        get_memory_layout(mci);
1667
1668        /* record ptr to the generic device */
1669        mci->pdev = &sbridge_dev->pdev[0]->dev;
1670
1671        /* add this new MC control structure to EDAC's list of MCs */
1672        if (unlikely(edac_mc_add_mc(mci))) {
1673                edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
1674                rc = -EINVAL;
1675                goto fail0;
1676        }
1677
1678        return 0;
1679
1680fail0:
1681        kfree(mci->ctl_name);
1682        edac_mc_free(mci);
1683        sbridge_dev->mci = NULL;
1684        return rc;
1685}
1686
1687/*
1688 *      sbridge_probe   Probe for ONE instance of device to see if it is
1689 *                      present.
1690 *      return:
1691 *              0 for FOUND a device
1692 *              < 0 for error code
1693 */
1694
1695static int sbridge_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1696{
1697        int rc;
1698        u8 mc, num_mc = 0;
1699        struct sbridge_dev *sbridge_dev;
1700
1701        /* get the pci devices we want to reserve for our use */
1702        mutex_lock(&sbridge_edac_lock);
1703
1704        /*
1705         * All memory controllers are allocated at the first pass.
1706         */
1707        if (unlikely(probed >= 1)) {
1708                mutex_unlock(&sbridge_edac_lock);
1709                return -ENODEV;
1710        }
1711        probed++;
1712
1713        rc = sbridge_get_all_devices(&num_mc);
1714        if (unlikely(rc < 0))
1715                goto fail0;
1716        mc = 0;
1717
1718        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list) {
1719                edac_dbg(0, "Registering MC#%d (%d of %d)\n",
1720                         mc, mc + 1, num_mc);
1721                sbridge_dev->mc = mc++;
1722                rc = sbridge_register_mci(sbridge_dev);
1723                if (unlikely(rc < 0))
1724                        goto fail1;
1725        }
1726
1727        sbridge_printk(KERN_INFO, "Driver loaded.\n");
1728
1729        mutex_unlock(&sbridge_edac_lock);
1730        return 0;
1731
1732fail1:
1733        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
1734                sbridge_unregister_mci(sbridge_dev);
1735
1736        sbridge_put_all_devices();
1737fail0:
1738        mutex_unlock(&sbridge_edac_lock);
1739        return rc;
1740}
1741
1742/*
1743 *      sbridge_remove  destructor for one instance of device
1744 *
1745 */
1746static void sbridge_remove(struct pci_dev *pdev)
1747{
1748        struct sbridge_dev *sbridge_dev;
1749
1750        edac_dbg(0, "\n");
1751
1752        /*
1753         * we have a trouble here: pdev value for removal will be wrong, since
1754         * it will point to the X58 register used to detect that the machine
1755         * is a Nehalem or upper design. However, due to the way several PCI
1756         * devices are grouped together to provide MC functionality, we need
1757         * to use a different method for releasing the devices
1758         */
1759
1760        mutex_lock(&sbridge_edac_lock);
1761
1762        if (unlikely(!probed)) {
1763                mutex_unlock(&sbridge_edac_lock);
1764                return;
1765        }
1766
1767        list_for_each_entry(sbridge_dev, &sbridge_edac_list, list)
1768                sbridge_unregister_mci(sbridge_dev);
1769
1770        /* Release PCI resources */
1771        sbridge_put_all_devices();
1772
1773        probed--;
1774
1775        mutex_unlock(&sbridge_edac_lock);
1776}
1777
1778MODULE_DEVICE_TABLE(pci, sbridge_pci_tbl);
1779
1780/*
1781 *      sbridge_driver  pci_driver structure for this module
1782 *
1783 */
1784static struct pci_driver sbridge_driver = {
1785        .name     = "sbridge_edac",
1786        .probe    = sbridge_probe,
1787        .remove   = sbridge_remove,
1788        .id_table = sbridge_pci_tbl,
1789};
1790
1791/*
1792 *      sbridge_init            Module entry function
1793 *                      Try to initialize this module for its devices
1794 */
1795static int __init sbridge_init(void)
1796{
1797        int pci_rc;
1798
1799        edac_dbg(2, "\n");
1800
1801        /* Ensure that the OPSTATE is set correctly for POLL or NMI */
1802        opstate_init();
1803
1804        pci_rc = pci_register_driver(&sbridge_driver);
1805
1806        if (pci_rc >= 0) {
1807                mce_register_decode_chain(&sbridge_mce_dec);
1808                return 0;
1809        }
1810
1811        sbridge_printk(KERN_ERR, "Failed to register device with error %d.\n",
1812                      pci_rc);
1813
1814        return pci_rc;
1815}
1816
1817/*
1818 *      sbridge_exit()  Module exit function
1819 *                      Unregister the driver
1820 */
1821static void __exit sbridge_exit(void)
1822{
1823        edac_dbg(2, "\n");
1824        pci_unregister_driver(&sbridge_driver);
1825        mce_unregister_decode_chain(&sbridge_mce_dec);
1826}
1827
1828module_init(sbridge_init);
1829module_exit(sbridge_exit);
1830
1831module_param(edac_op_state, int, 0444);
1832MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
1833
1834MODULE_LICENSE("GPL");
1835MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
1836MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
1837MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge memory controllers - "
1838                   SBRIDGE_REVISION);
1839
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.