linux/drivers/edac/i7core_edac.c
<<
>>
Prefs
   1/* Intel i7 core/Nehalem Memory Controller kernel module
   2 *
   3 * This driver supports the memory controllers found on the Intel
   4 * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5 * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6 * and Westmere-EP.
   7 *
   8 * This file may be distributed under the terms of the
   9 * GNU General Public License version 2 only.
  10 *
  11 * Copyright (c) 2009-2010 by:
  12 *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13 *
  14 * Red Hat Inc. http://www.redhat.com
  15 *
  16 * Forked and adapted from the i5400_edac driver
  17 *
  18 * Based on the following public Intel datasheets:
  19 * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20 * Datasheet, Volume 2:
  21 *      http://download.intel.com/design/processor/datashts/320835.pdf
  22 * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23 *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24 * also available at:
  25 *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26 */
  27
  28#include <linux/module.h>
  29#include <linux/init.h>
  30#include <linux/pci.h>
  31#include <linux/pci_ids.h>
  32#include <linux/slab.h>
  33#include <linux/delay.h>
  34#include <linux/edac.h>
  35#include <linux/mmzone.h>
  36#include <linux/edac_mce.h>
  37#include <linux/smp.h>
  38#include <asm/processor.h>
  39
  40#include "edac_core.h"
  41
  42/* Static vars */
  43static LIST_HEAD(i7core_edac_list);
  44static DEFINE_MUTEX(i7core_edac_lock);
  45static int probed;
  46
  47static int use_pci_fixup;
  48module_param(use_pci_fixup, int, 0444);
  49MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  50/*
  51 * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  52 * registers start at bus 255, and are not reported by BIOS.
  53 * We currently find devices with only 2 sockets. In order to support more QPI
  54 * Quick Path Interconnect, just increment this number.
  55 */
  56#define MAX_SOCKET_BUSES        2
  57
  58
  59/*
  60 * Alter this version for the module when modifications are made
  61 */
  62#define I7CORE_REVISION    " Ver: 1.0.0 " __DATE__
  63#define EDAC_MOD_STR      "i7core_edac"
  64
  65/*
  66 * Debug macros
  67 */
  68#define i7core_printk(level, fmt, arg...)                       \
  69        edac_printk(level, "i7core", fmt, ##arg)
  70
  71#define i7core_mc_printk(mci, level, fmt, arg...)               \
  72        edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  73
  74/*
  75 * i7core Memory Controller Registers
  76 */
  77
  78        /* OFFSETS for Device 0 Function 0 */
  79
  80#define MC_CFG_CONTROL  0x90
  81
  82        /* OFFSETS for Device 3 Function 0 */
  83
  84#define MC_CONTROL      0x48
  85#define MC_STATUS       0x4c
  86#define MC_MAX_DOD      0x64
  87
  88/*
  89 * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  90 * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  91 */
  92
  93#define MC_TEST_ERR_RCV1        0x60
  94  #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  95
  96#define MC_TEST_ERR_RCV0        0x64
  97  #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
  98  #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100/* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 101#define MC_COR_ECC_CNT_0        0x80
 102#define MC_COR_ECC_CNT_1        0x84
 103#define MC_COR_ECC_CNT_2        0x88
 104#define MC_COR_ECC_CNT_3        0x8c
 105#define MC_COR_ECC_CNT_4        0x90
 106#define MC_COR_ECC_CNT_5        0x94
 107
 108#define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 109#define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 110
 111
 112        /* OFFSETS for Devices 4,5 and 6 Function 0 */
 113
 114#define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 115  #define THREE_DIMMS_PRESENT           (1 << 24)
 116  #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 117  #define QUAD_RANK_PRESENT             (1 << 22)
 118  #define REGISTERED_DIMM               (1 << 15)
 119
 120#define MC_CHANNEL_MAPPER       0x60
 121  #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 122  #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 123
 124#define MC_CHANNEL_RANK_PRESENT 0x7c
 125  #define RANK_PRESENT_MASK             0xffff
 126
 127#define MC_CHANNEL_ADDR_MATCH   0xf0
 128#define MC_CHANNEL_ERROR_MASK   0xf8
 129#define MC_CHANNEL_ERROR_INJECT 0xfc
 130  #define INJECT_ADDR_PARITY    0x10
 131  #define INJECT_ECC            0x08
 132  #define MASK_CACHELINE        0x06
 133  #define MASK_FULL_CACHELINE   0x06
 134  #define MASK_MSB32_CACHELINE  0x04
 135  #define MASK_LSB32_CACHELINE  0x02
 136  #define NO_MASK_CACHELINE     0x00
 137  #define REPEAT_EN             0x01
 138
 139        /* OFFSETS for Devices 4,5 and 6 Function 1 */
 140
 141#define MC_DOD_CH_DIMM0         0x48
 142#define MC_DOD_CH_DIMM1         0x4c
 143#define MC_DOD_CH_DIMM2         0x50
 144  #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 145  #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 146  #define DIMM_PRESENT_MASK     (1 << 9)
 147  #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 148  #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 149  #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 150  #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 151  #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 152  #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 153  #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 154  #define MC_DOD_NUMCOL_MASK            3
 155  #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 156
 157#define MC_RANK_PRESENT         0x7c
 158
 159#define MC_SAG_CH_0     0x80
 160#define MC_SAG_CH_1     0x84
 161#define MC_SAG_CH_2     0x88
 162#define MC_SAG_CH_3     0x8c
 163#define MC_SAG_CH_4     0x90
 164#define MC_SAG_CH_5     0x94
 165#define MC_SAG_CH_6     0x98
 166#define MC_SAG_CH_7     0x9c
 167
 168#define MC_RIR_LIMIT_CH_0       0x40
 169#define MC_RIR_LIMIT_CH_1       0x44
 170#define MC_RIR_LIMIT_CH_2       0x48
 171#define MC_RIR_LIMIT_CH_3       0x4C
 172#define MC_RIR_LIMIT_CH_4       0x50
 173#define MC_RIR_LIMIT_CH_5       0x54
 174#define MC_RIR_LIMIT_CH_6       0x58
 175#define MC_RIR_LIMIT_CH_7       0x5C
 176#define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 177
 178#define MC_RIR_WAY_CH           0x80
 179  #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 180  #define MC_RIR_WAY_RANK_MASK          0x7
 181
 182/*
 183 * i7core structs
 184 */
 185
 186#define NUM_CHANS 3
 187#define MAX_DIMMS 3             /* Max DIMMS per channel */
 188#define MAX_MCR_FUNC  4
 189#define MAX_CHAN_FUNC 3
 190
 191struct i7core_info {
 192        u32     mc_control;
 193        u32     mc_status;
 194        u32     max_dod;
 195        u32     ch_map;
 196};
 197
 198
 199struct i7core_inject {
 200        int     enable;
 201
 202        u32     section;
 203        u32     type;
 204        u32     eccmask;
 205
 206        /* Error address mask */
 207        int channel, dimm, rank, bank, page, col;
 208};
 209
 210struct i7core_channel {
 211        u32             ranks;
 212        u32             dimms;
 213};
 214
 215struct pci_id_descr {
 216        int                     dev;
 217        int                     func;
 218        int                     dev_id;
 219        int                     optional;
 220};
 221
 222struct pci_id_table {
 223        const struct pci_id_descr       *descr;
 224        int                             n_devs;
 225};
 226
 227struct i7core_dev {
 228        struct list_head        list;
 229        u8                      socket;
 230        struct pci_dev          **pdev;
 231        int                     n_devs;
 232        struct mem_ctl_info     *mci;
 233};
 234
 235struct i7core_pvt {
 236        struct pci_dev  *pci_noncore;
 237        struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 238        struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 239
 240        struct i7core_dev *i7core_dev;
 241
 242        struct i7core_info      info;
 243        struct i7core_inject    inject;
 244        struct i7core_channel   channel[NUM_CHANS];
 245
 246        int             ce_count_available;
 247        int             csrow_map[NUM_CHANS][MAX_DIMMS];
 248
 249                        /* ECC corrected errors counts per udimm */
 250        unsigned long   udimm_ce_count[MAX_DIMMS];
 251        int             udimm_last_ce_count[MAX_DIMMS];
 252                        /* ECC corrected errors counts per rdimm */
 253        unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 254        int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 255
 256        unsigned int    is_registered;
 257
 258        /* mcelog glue */
 259        struct edac_mce         edac_mce;
 260
 261        /* Fifo double buffers */
 262        struct mce              mce_entry[MCE_LOG_LEN];
 263        struct mce              mce_outentry[MCE_LOG_LEN];
 264
 265        /* Fifo in/out counters */
 266        unsigned                mce_in, mce_out;
 267
 268        /* Count indicator to show errors not got */
 269        unsigned                mce_overrun;
 270
 271        /* Struct to control EDAC polling */
 272        struct edac_pci_ctl_info *i7core_pci;
 273};
 274
 275#define PCI_DESCR(device, function, device_id)  \
 276        .dev = (device),                        \
 277        .func = (function),                     \
 278        .dev_id = (device_id)
 279
 280static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 281                /* Memory controller */
 282        { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 283        { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 284
 285                /* Exists only for RDIMM */
 286        { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 287        { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 288
 289                /* Channel 0 */
 290        { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 291        { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 292        { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 293        { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 294
 295                /* Channel 1 */
 296        { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 297        { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 298        { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 299        { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 300
 301                /* Channel 2 */
 302        { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 303        { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 304        { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 305        { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 306};
 307
 308static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 309        { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 310        { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 311        { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 312
 313        { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 314        { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 315        { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 316        { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 317
 318        { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 319        { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 320        { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 321        { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 322};
 323
 324static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 325                /* Memory controller */
 326        { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 327        { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 328                        /* Exists only for RDIMM */
 329        { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 330        { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 331
 332                /* Channel 0 */
 333        { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 334        { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 335        { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 336        { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 337
 338                /* Channel 1 */
 339        { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 340        { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 341        { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 342        { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 343
 344                /* Channel 2 */
 345        { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 346        { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 347        { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 348        { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 349};
 350
 351#define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 352static const struct pci_id_table pci_dev_table[] = {
 353        PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 354        PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 355        PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 356        {0,}                    /* 0 terminated list. */
 357};
 358
 359/*
 360 *      pci_device_id   table for which devices we are looking for
 361 */
 362static const struct pci_device_id i7core_pci_tbl[] __devinitdata = {
 363        {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 364        {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 365        {0,}                    /* 0 terminated list. */
 366};
 367
 368/****************************************************************************
 369                        Anciliary status routines
 370 ****************************************************************************/
 371
 372        /* MC_CONTROL bits */
 373#define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 374#define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 375
 376        /* MC_STATUS bits */
 377#define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 378#define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 379
 380        /* MC_MAX_DOD read functions */
 381static inline int numdimms(u32 dimms)
 382{
 383        return (dimms & 0x3) + 1;
 384}
 385
 386static inline int numrank(u32 rank)
 387{
 388        static int ranks[4] = { 1, 2, 4, -EINVAL };
 389
 390        return ranks[rank & 0x3];
 391}
 392
 393static inline int numbank(u32 bank)
 394{
 395        static int banks[4] = { 4, 8, 16, -EINVAL };
 396
 397        return banks[bank & 0x3];
 398}
 399
 400static inline int numrow(u32 row)
 401{
 402        static int rows[8] = {
 403                1 << 12, 1 << 13, 1 << 14, 1 << 15,
 404                1 << 16, -EINVAL, -EINVAL, -EINVAL,
 405        };
 406
 407        return rows[row & 0x7];
 408}
 409
 410static inline int numcol(u32 col)
 411{
 412        static int cols[8] = {
 413                1 << 10, 1 << 11, 1 << 12, -EINVAL,
 414        };
 415        return cols[col & 0x3];
 416}
 417
 418static struct i7core_dev *get_i7core_dev(u8 socket)
 419{
 420        struct i7core_dev *i7core_dev;
 421
 422        list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 423                if (i7core_dev->socket == socket)
 424                        return i7core_dev;
 425        }
 426
 427        return NULL;
 428}
 429
 430static struct i7core_dev *alloc_i7core_dev(u8 socket,
 431                                           const struct pci_id_table *table)
 432{
 433        struct i7core_dev *i7core_dev;
 434
 435        i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 436        if (!i7core_dev)
 437                return NULL;
 438
 439        i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs,
 440                                   GFP_KERNEL);
 441        if (!i7core_dev->pdev) {
 442                kfree(i7core_dev);
 443                return NULL;
 444        }
 445
 446        i7core_dev->socket = socket;
 447        i7core_dev->n_devs = table->n_devs;
 448        list_add_tail(&i7core_dev->list, &i7core_edac_list);
 449
 450        return i7core_dev;
 451}
 452
 453static void free_i7core_dev(struct i7core_dev *i7core_dev)
 454{
 455        list_del(&i7core_dev->list);
 456        kfree(i7core_dev->pdev);
 457        kfree(i7core_dev);
 458}
 459
 460/****************************************************************************
 461                        Memory check routines
 462 ****************************************************************************/
 463static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
 464                                          unsigned func)
 465{
 466        struct i7core_dev *i7core_dev = get_i7core_dev(socket);
 467        int i;
 468
 469        if (!i7core_dev)
 470                return NULL;
 471
 472        for (i = 0; i < i7core_dev->n_devs; i++) {
 473                if (!i7core_dev->pdev[i])
 474                        continue;
 475
 476                if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
 477                    PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
 478                        return i7core_dev->pdev[i];
 479                }
 480        }
 481
 482        return NULL;
 483}
 484
 485/**
 486 * i7core_get_active_channels() - gets the number of channels and csrows
 487 * @socket:     Quick Path Interconnect socket
 488 * @channels:   Number of channels that will be returned
 489 * @csrows:     Number of csrows found
 490 *
 491 * Since EDAC core needs to know in advance the number of available channels
 492 * and csrows, in order to allocate memory for csrows/channels, it is needed
 493 * to run two similar steps. At the first step, implemented on this function,
 494 * it checks the number of csrows/channels present at one socket.
 495 * this is used in order to properly allocate the size of mci components.
 496 *
 497 * It should be noticed that none of the current available datasheets explain
 498 * or even mention how csrows are seen by the memory controller. So, we need
 499 * to add a fake description for csrows.
 500 * So, this driver is attributing one DIMM memory for one csrow.
 501 */
 502static int i7core_get_active_channels(const u8 socket, unsigned *channels,
 503                                      unsigned *csrows)
 504{
 505        struct pci_dev *pdev = NULL;
 506        int i, j;
 507        u32 status, control;
 508
 509        *channels = 0;
 510        *csrows = 0;
 511
 512        pdev = get_pdev_slot_func(socket, 3, 0);
 513        if (!pdev) {
 514                i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
 515                              socket);
 516                return -ENODEV;
 517        }
 518
 519        /* Device 3 function 0 reads */
 520        pci_read_config_dword(pdev, MC_STATUS, &status);
 521        pci_read_config_dword(pdev, MC_CONTROL, &control);
 522
 523        for (i = 0; i < NUM_CHANS; i++) {
 524                u32 dimm_dod[3];
 525                /* Check if the channel is active */
 526                if (!(control & (1 << (8 + i))))
 527                        continue;
 528
 529                /* Check if the channel is disabled */
 530                if (status & (1 << i))
 531                        continue;
 532
 533                pdev = get_pdev_slot_func(socket, i + 4, 1);
 534                if (!pdev) {
 535                        i7core_printk(KERN_ERR, "Couldn't find socket %d "
 536                                                "fn %d.%d!!!\n",
 537                                                socket, i + 4, 1);
 538                        return -ENODEV;
 539                }
 540                /* Devices 4-6 function 1 */
 541                pci_read_config_dword(pdev,
 542                                MC_DOD_CH_DIMM0, &dimm_dod[0]);
 543                pci_read_config_dword(pdev,
 544                                MC_DOD_CH_DIMM1, &dimm_dod[1]);
 545                pci_read_config_dword(pdev,
 546                                MC_DOD_CH_DIMM2, &dimm_dod[2]);
 547
 548                (*channels)++;
 549
 550                for (j = 0; j < 3; j++) {
 551                        if (!DIMM_PRESENT(dimm_dod[j]))
 552                                continue;
 553                        (*csrows)++;
 554                }
 555        }
 556
 557        debugf0("Number of active channels on socket %d: %d\n",
 558                socket, *channels);
 559
 560        return 0;
 561}
 562
 563static int get_dimm_config(const struct mem_ctl_info *mci)
 564{
 565        struct i7core_pvt *pvt = mci->pvt_info;
 566        struct csrow_info *csr;
 567        struct pci_dev *pdev;
 568        int i, j;
 569        int csrow = 0;
 570        unsigned long last_page = 0;
 571        enum edac_type mode;
 572        enum mem_type mtype;
 573
 574        /* Get data from the MC register, function 0 */
 575        pdev = pvt->pci_mcr[0];
 576        if (!pdev)
 577                return -ENODEV;
 578
 579        /* Device 3 function 0 reads */
 580        pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 581        pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 582        pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 583        pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 584
 585        debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 586                pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 587                pvt->info.max_dod, pvt->info.ch_map);
 588
 589        if (ECC_ENABLED(pvt)) {
 590                debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 591                if (ECCx8(pvt))
 592                        mode = EDAC_S8ECD8ED;
 593                else
 594                        mode = EDAC_S4ECD4ED;
 595        } else {
 596                debugf0("ECC disabled\n");
 597                mode = EDAC_NONE;
 598        }
 599
 600        /* FIXME: need to handle the error codes */
 601        debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 602                "x%x x 0x%x\n",
 603                numdimms(pvt->info.max_dod),
 604                numrank(pvt->info.max_dod >> 2),
 605                numbank(pvt->info.max_dod >> 4),
 606                numrow(pvt->info.max_dod >> 6),
 607                numcol(pvt->info.max_dod >> 9));
 608
 609        for (i = 0; i < NUM_CHANS; i++) {
 610                u32 data, dimm_dod[3], value[8];
 611
 612                if (!pvt->pci_ch[i][0])
 613                        continue;
 614
 615                if (!CH_ACTIVE(pvt, i)) {
 616                        debugf0("Channel %i is not active\n", i);
 617                        continue;
 618                }
 619                if (CH_DISABLED(pvt, i)) {
 620                        debugf0("Channel %i is disabled\n", i);
 621                        continue;
 622                }
 623
 624                /* Devices 4-6 function 0 */
 625                pci_read_config_dword(pvt->pci_ch[i][0],
 626                                MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 627
 628                pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
 629                                                4 : 2;
 630
 631                if (data & REGISTERED_DIMM)
 632                        mtype = MEM_RDDR3;
 633                else
 634                        mtype = MEM_DDR3;
 635#if 0
 636                if (data & THREE_DIMMS_PRESENT)
 637                        pvt->channel[i].dimms = 3;
 638                else if (data & SINGLE_QUAD_RANK_PRESENT)
 639                        pvt->channel[i].dimms = 1;
 640                else
 641                        pvt->channel[i].dimms = 2;
 642#endif
 643
 644                /* Devices 4-6 function 1 */
 645                pci_read_config_dword(pvt->pci_ch[i][1],
 646                                MC_DOD_CH_DIMM0, &dimm_dod[0]);
 647                pci_read_config_dword(pvt->pci_ch[i][1],
 648                                MC_DOD_CH_DIMM1, &dimm_dod[1]);
 649                pci_read_config_dword(pvt->pci_ch[i][1],
 650                                MC_DOD_CH_DIMM2, &dimm_dod[2]);
 651
 652                debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 653                        "%d ranks, %cDIMMs\n",
 654                        i,
 655                        RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 656                        data,
 657                        pvt->channel[i].ranks,
 658                        (data & REGISTERED_DIMM) ? 'R' : 'U');
 659
 660                for (j = 0; j < 3; j++) {
 661                        u32 banks, ranks, rows, cols;
 662                        u32 size, npages;
 663
 664                        if (!DIMM_PRESENT(dimm_dod[j]))
 665                                continue;
 666
 667                        banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 668                        ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 669                        rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 670                        cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 671
 672                        /* DDR3 has 8 I/O banks */
 673                        size = (rows * cols * banks * ranks) >> (20 - 3);
 674
 675                        pvt->channel[i].dimms++;
 676
 677                        debugf0("\tdimm %d %d Mb offset: %x, "
 678                                "bank: %d, rank: %d, row: %#x, col: %#x\n",
 679                                j, size,
 680                                RANKOFFSET(dimm_dod[j]),
 681                                banks, ranks, rows, cols);
 682
 683                        npages = MiB_TO_PAGES(size);
 684
 685                        csr = &mci->csrows[csrow];
 686                        csr->first_page = last_page + 1;
 687                        last_page += npages;
 688                        csr->last_page = last_page;
 689                        csr->nr_pages = npages;
 690
 691                        csr->page_mask = 0;
 692                        csr->grain = 8;
 693                        csr->csrow_idx = csrow;
 694                        csr->nr_channels = 1;
 695
 696                        csr->channels[0].chan_idx = i;
 697                        csr->channels[0].ce_count = 0;
 698
 699                        pvt->csrow_map[i][j] = csrow;
 700
 701                        switch (banks) {
 702                        case 4:
 703                                csr->dtype = DEV_X4;
 704                                break;
 705                        case 8:
 706                                csr->dtype = DEV_X8;
 707                                break;
 708                        case 16:
 709                                csr->dtype = DEV_X16;
 710                                break;
 711                        default:
 712                                csr->dtype = DEV_UNKNOWN;
 713                        }
 714
 715                        csr->edac_mode = mode;
 716                        csr->mtype = mtype;
 717
 718                        csrow++;
 719                }
 720
 721                pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 722                pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 723                pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 724                pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 725                pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 726                pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 727                pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 728                pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 729                debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 730                for (j = 0; j < 8; j++)
 731                        debugf1("\t\t%#x\t%#x\t%#x\n",
 732                                (value[j] >> 27) & 0x1,
 733                                (value[j] >> 24) & 0x7,
 734                                (value[j] && ((1 << 24) - 1)));
 735        }
 736
 737        return 0;
 738}
 739
 740/****************************************************************************
 741                        Error insertion routines
 742 ****************************************************************************/
 743
 744/* The i7core has independent error injection features per channel.
 745   However, to have a simpler code, we don't allow enabling error injection
 746   on more than one channel.
 747   Also, since a change at an inject parameter will be applied only at enable,
 748   we're disabling error injection on all write calls to the sysfs nodes that
 749   controls the error code injection.
 750 */
 751static int disable_inject(const struct mem_ctl_info *mci)
 752{
 753        struct i7core_pvt *pvt = mci->pvt_info;
 754
 755        pvt->inject.enable = 0;
 756
 757        if (!pvt->pci_ch[pvt->inject.channel][0])
 758                return -ENODEV;
 759
 760        pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 761                                MC_CHANNEL_ERROR_INJECT, 0);
 762
 763        return 0;
 764}
 765
 766/*
 767 * i7core inject inject.section
 768 *
 769 *      accept and store error injection inject.section value
 770 *      bit 0 - refers to the lower 32-byte half cacheline
 771 *      bit 1 - refers to the upper 32-byte half cacheline
 772 */
 773static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
 774                                           const char *data, size_t count)
 775{
 776        struct i7core_pvt *pvt = mci->pvt_info;
 777        unsigned long value;
 778        int rc;
 779
 780        if (pvt->inject.enable)
 781                disable_inject(mci);
 782
 783        rc = strict_strtoul(data, 10, &value);
 784        if ((rc < 0) || (value > 3))
 785                return -EIO;
 786
 787        pvt->inject.section = (u32) value;
 788        return count;
 789}
 790
 791static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
 792                                              char *data)
 793{
 794        struct i7core_pvt *pvt = mci->pvt_info;
 795        return sprintf(data, "0x%08x\n", pvt->inject.section);
 796}
 797
 798/*
 799 * i7core inject.type
 800 *
 801 *      accept and store error injection inject.section value
 802 *      bit 0 - repeat enable - Enable error repetition
 803 *      bit 1 - inject ECC error
 804 *      bit 2 - inject parity error
 805 */
 806static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
 807                                        const char *data, size_t count)
 808{
 809        struct i7core_pvt *pvt = mci->pvt_info;
 810        unsigned long value;
 811        int rc;
 812
 813        if (pvt->inject.enable)
 814                disable_inject(mci);
 815
 816        rc = strict_strtoul(data, 10, &value);
 817        if ((rc < 0) || (value > 7))
 818                return -EIO;
 819
 820        pvt->inject.type = (u32) value;
 821        return count;
 822}
 823
 824static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
 825                                              char *data)
 826{
 827        struct i7core_pvt *pvt = mci->pvt_info;
 828        return sprintf(data, "0x%08x\n", pvt->inject.type);
 829}
 830
 831/*
 832 * i7core_inject_inject.eccmask_store
 833 *
 834 * The type of error (UE/CE) will depend on the inject.eccmask value:
 835 *   Any bits set to a 1 will flip the corresponding ECC bit
 836 *   Correctable errors can be injected by flipping 1 bit or the bits within
 837 *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 838 *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 839 *   uncorrectable error to be injected.
 840 */
 841static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
 842                                        const char *data, size_t count)
 843{
 844        struct i7core_pvt *pvt = mci->pvt_info;
 845        unsigned long value;
 846        int rc;
 847
 848        if (pvt->inject.enable)
 849                disable_inject(mci);
 850
 851        rc = strict_strtoul(data, 10, &value);
 852        if (rc < 0)
 853                return -EIO;
 854
 855        pvt->inject.eccmask = (u32) value;
 856        return count;
 857}
 858
 859static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
 860                                              char *data)
 861{
 862        struct i7core_pvt *pvt = mci->pvt_info;
 863        return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 864}
 865
 866/*
 867 * i7core_addrmatch
 868 *
 869 * The type of error (UE/CE) will depend on the inject.eccmask value:
 870 *   Any bits set to a 1 will flip the corresponding ECC bit
 871 *   Correctable errors can be injected by flipping 1 bit or the bits within
 872 *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 873 *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 874 *   uncorrectable error to be injected.
 875 */
 876
 877#define DECLARE_ADDR_MATCH(param, limit)                        \
 878static ssize_t i7core_inject_store_##param(                     \
 879                struct mem_ctl_info *mci,                       \
 880                const char *data, size_t count)                 \
 881{                                                               \
 882        struct i7core_pvt *pvt;                                 \
 883        long value;                                             \
 884        int rc;                                                 \
 885                                                                \
 886        debugf1("%s()\n", __func__);                            \
 887        pvt = mci->pvt_info;                                    \
 888                                                                \
 889        if (pvt->inject.enable)                                 \
 890                disable_inject(mci);                            \
 891                                                                \
 892        if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 893                value = -1;                                     \
 894        else {                                                  \
 895                rc = strict_strtoul(data, 10, &value);          \
 896                if ((rc < 0) || (value >= limit))               \
 897                        return -EIO;                            \
 898        }                                                       \
 899                                                                \
 900        pvt->inject.param = value;                              \
 901                                                                \
 902        return count;                                           \
 903}                                                               \
 904                                                                \
 905static ssize_t i7core_inject_show_##param(                      \
 906                struct mem_ctl_info *mci,                       \
 907                char *data)                                     \
 908{                                                               \
 909        struct i7core_pvt *pvt;                                 \
 910                                                                \
 911        pvt = mci->pvt_info;                                    \
 912        debugf1("%s() pvt=%p\n", __func__, pvt);                \
 913        if (pvt->inject.param < 0)                              \
 914                return sprintf(data, "any\n");                  \
 915        else                                                    \
 916                return sprintf(data, "%d\n", pvt->inject.param);\
 917}
 918
 919#define ATTR_ADDR_MATCH(param)                                  \
 920        {                                                       \
 921                .attr = {                                       \
 922                        .name = #param,                         \
 923                        .mode = (S_IRUGO | S_IWUSR)             \
 924                },                                              \
 925                .show  = i7core_inject_show_##param,            \
 926                .store = i7core_inject_store_##param,           \
 927        }
 928
 929DECLARE_ADDR_MATCH(channel, 3);
 930DECLARE_ADDR_MATCH(dimm, 3);
 931DECLARE_ADDR_MATCH(rank, 4);
 932DECLARE_ADDR_MATCH(bank, 32);
 933DECLARE_ADDR_MATCH(page, 0x10000);
 934DECLARE_ADDR_MATCH(col, 0x4000);
 935
 936static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 937{
 938        u32 read;
 939        int count;
 940
 941        debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 942                dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 943                where, val);
 944
 945        for (count = 0; count < 10; count++) {
 946                if (count)
 947                        msleep(100);
 948                pci_write_config_dword(dev, where, val);
 949                pci_read_config_dword(dev, where, &read);
 950
 951                if (read == val)
 952                        return 0;
 953        }
 954
 955        i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 956                "write=%08x. Read=%08x\n",
 957                dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 958                where, val, read);
 959
 960        return -EINVAL;
 961}
 962
 963/*
 964 * This routine prepares the Memory Controller for error injection.
 965 * The error will be injected when some process tries to write to the
 966 * memory that matches the given criteria.
 967 * The criteria can be set in terms of a mask where dimm, rank, bank, page
 968 * and col can be specified.
 969 * A -1 value for any of the mask items will make the MCU to ignore
 970 * that matching criteria for error injection.
 971 *
 972 * It should be noticed that the error will only happen after a write operation
 973 * on a memory that matches the condition. if REPEAT_EN is not enabled at
 974 * inject mask, then it will produce just one error. Otherwise, it will repeat
 975 * until the injectmask would be cleaned.
 976 *
 977 * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 978 *    is reliable enough to check if the MC is using the
 979 *    three channels. However, this is not clear at the datasheet.
 980 */
 981static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
 982                                       const char *data, size_t count)
 983{
 984        struct i7core_pvt *pvt = mci->pvt_info;
 985        u32 injectmask;
 986        u64 mask = 0;
 987        int  rc;
 988        long enable;
 989
 990        if (!pvt->pci_ch[pvt->inject.channel][0])
 991                return 0;
 992
 993        rc = strict_strtoul(data, 10, &enable);
 994        if ((rc < 0))
 995                return 0;
 996
 997        if (enable) {
 998                pvt->inject.enable = 1;
 999        } else {
1000                disable_inject(mci);
1001                return count;
1002        }
1003
1004        /* Sets pvt->inject.dimm mask */
1005        if (pvt->inject.dimm < 0)
1006                mask |= 1LL << 41;
1007        else {
1008                if (pvt->channel[pvt->inject.channel].dimms > 2)
1009                        mask |= (pvt->inject.dimm & 0x3LL) << 35;
1010                else
1011                        mask |= (pvt->inject.dimm & 0x1LL) << 36;
1012        }
1013
1014        /* Sets pvt->inject.rank mask */
1015        if (pvt->inject.rank < 0)
1016                mask |= 1LL << 40;
1017        else {
1018                if (pvt->channel[pvt->inject.channel].dimms > 2)
1019                        mask |= (pvt->inject.rank & 0x1LL) << 34;
1020                else
1021                        mask |= (pvt->inject.rank & 0x3LL) << 34;
1022        }
1023
1024        /* Sets pvt->inject.bank mask */
1025        if (pvt->inject.bank < 0)
1026                mask |= 1LL << 39;
1027        else
1028                mask |= (pvt->inject.bank & 0x15LL) << 30;
1029
1030        /* Sets pvt->inject.page mask */
1031        if (pvt->inject.page < 0)
1032                mask |= 1LL << 38;
1033        else
1034                mask |= (pvt->inject.page & 0xffff) << 14;
1035
1036        /* Sets pvt->inject.column mask */
1037        if (pvt->inject.col < 0)
1038                mask |= 1LL << 37;
1039        else
1040                mask |= (pvt->inject.col & 0x3fff);
1041
1042        /*
1043         * bit    0: REPEAT_EN
1044         * bits 1-2: MASK_HALF_CACHELINE
1045         * bit    3: INJECT_ECC
1046         * bit    4: INJECT_ADDR_PARITY
1047         */
1048
1049        injectmask = (pvt->inject.type & 1) |
1050                     (pvt->inject.section & 0x3) << 1 |
1051                     (pvt->inject.type & 0x6) << (3 - 1);
1052
1053        /* Unlock writes to registers - this register is write only */
1054        pci_write_config_dword(pvt->pci_noncore,
1055                               MC_CFG_CONTROL, 0x2);
1056
1057        write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1058                               MC_CHANNEL_ADDR_MATCH, mask);
1059        write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1060                               MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1061
1062        write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1063                               MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1064
1065        write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1066                               MC_CHANNEL_ERROR_INJECT, injectmask);
1067
1068        /*
1069         * This is something undocumented, based on my tests
1070         * Without writing 8 to this register, errors aren't injected. Not sure
1071         * why.
1072         */
1073        pci_write_config_dword(pvt->pci_noncore,
1074                               MC_CFG_CONTROL, 8);
1075
1076        debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1077                " inject 0x%08x\n",
1078                mask, pvt->inject.eccmask, injectmask);
1079
1080
1081        return count;
1082}
1083
1084static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1085                                        char *data)
1086{
1087        struct i7core_pvt *pvt = mci->pvt_info;
1088        u32 injectmask;
1089
1090        if (!pvt->pci_ch[pvt->inject.channel][0])
1091                return 0;
1092
1093        pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1094                               MC_CHANNEL_ERROR_INJECT, &injectmask);
1095
1096        debugf0("Inject error read: 0x%018x\n", injectmask);
1097
1098        if (injectmask & 0x0c)
1099                pvt->inject.enable = 1;
1100
1101        return sprintf(data, "%d\n", pvt->inject.enable);
1102}
1103
1104#define DECLARE_COUNTER(param)                                  \
1105static ssize_t i7core_show_counter_##param(                     \
1106                struct mem_ctl_info *mci,                       \
1107                char *data)                                     \
1108{                                                               \
1109        struct i7core_pvt *pvt = mci->pvt_info;                 \
1110                                                                \
1111        debugf1("%s() \n", __func__);                           \
1112        if (!pvt->ce_count_available || (pvt->is_registered))   \
1113                return sprintf(data, "data unavailable\n");     \
1114        return sprintf(data, "%lu\n",                           \
1115                        pvt->udimm_ce_count[param]);            \
1116}
1117
1118#define ATTR_COUNTER(param)                                     \
1119        {                                                       \
1120                .attr = {                                       \
1121                        .name = __stringify(udimm##param),      \
1122                        .mode = (S_IRUGO | S_IWUSR)             \
1123                },                                              \
1124                .show  = i7core_show_counter_##param            \
1125        }
1126
1127DECLARE_COUNTER(0);
1128DECLARE_COUNTER(1);
1129DECLARE_COUNTER(2);
1130
1131/*
1132 * Sysfs struct
1133 */
1134
1135static const struct mcidev_sysfs_attribute i7core_addrmatch_attrs[] = {
1136        ATTR_ADDR_MATCH(channel),
1137        ATTR_ADDR_MATCH(dimm),
1138        ATTR_ADDR_MATCH(rank),
1139        ATTR_ADDR_MATCH(bank),
1140        ATTR_ADDR_MATCH(page),
1141        ATTR_ADDR_MATCH(col),
1142        { } /* End of list */
1143};
1144
1145static const struct mcidev_sysfs_group i7core_inject_addrmatch = {
1146        .name  = "inject_addrmatch",
1147        .mcidev_attr = i7core_addrmatch_attrs,
1148};
1149
1150static const struct mcidev_sysfs_attribute i7core_udimm_counters_attrs[] = {
1151        ATTR_COUNTER(0),
1152        ATTR_COUNTER(1),
1153        ATTR_COUNTER(2),
1154        { .attr = { .name = NULL } }
1155};
1156
1157static const struct mcidev_sysfs_group i7core_udimm_counters = {
1158        .name  = "all_channel_counts",
1159        .mcidev_attr = i7core_udimm_counters_attrs,
1160};
1161
1162static const struct mcidev_sysfs_attribute i7core_sysfs_rdimm_attrs[] = {
1163        {
1164                .attr = {
1165                        .name = "inject_section",
1166                        .mode = (S_IRUGO | S_IWUSR)
1167                },
1168                .show  = i7core_inject_section_show,
1169                .store = i7core_inject_section_store,
1170        }, {
1171                .attr = {
1172                        .name = "inject_type",
1173                        .mode = (S_IRUGO | S_IWUSR)
1174                },
1175                .show  = i7core_inject_type_show,
1176                .store = i7core_inject_type_store,
1177        }, {
1178                .attr = {
1179                        .name = "inject_eccmask",
1180                        .mode = (S_IRUGO | S_IWUSR)
1181                },
1182                .show  = i7core_inject_eccmask_show,
1183                .store = i7core_inject_eccmask_store,
1184        }, {
1185                .grp = &i7core_inject_addrmatch,
1186        }, {
1187                .attr = {
1188                        .name = "inject_enable",
1189                        .mode = (S_IRUGO | S_IWUSR)
1190                },
1191                .show  = i7core_inject_enable_show,
1192                .store = i7core_inject_enable_store,
1193        },
1194        { }     /* End of list */
1195};
1196
1197static const struct mcidev_sysfs_attribute i7core_sysfs_udimm_attrs[] = {
1198        {
1199                .attr = {
1200                        .name = "inject_section",
1201                        .mode = (S_IRUGO | S_IWUSR)
1202                },
1203                .show  = i7core_inject_section_show,
1204                .store = i7core_inject_section_store,
1205        }, {
1206                .attr = {
1207                        .name = "inject_type",
1208                        .mode = (S_IRUGO | S_IWUSR)
1209                },
1210                .show  = i7core_inject_type_show,
1211                .store = i7core_inject_type_store,
1212        }, {
1213                .attr = {
1214                        .name = "inject_eccmask",
1215                        .mode = (S_IRUGO | S_IWUSR)
1216                },
1217                .show  = i7core_inject_eccmask_show,
1218                .store = i7core_inject_eccmask_store,
1219        }, {
1220                .grp = &i7core_inject_addrmatch,
1221        }, {
1222                .attr = {
1223                        .name = "inject_enable",
1224                        .mode = (S_IRUGO | S_IWUSR)
1225                },
1226                .show  = i7core_inject_enable_show,
1227                .store = i7core_inject_enable_store,
1228        }, {
1229                .grp = &i7core_udimm_counters,
1230        },
1231        { }     /* End of list */
1232};
1233
1234/****************************************************************************
1235        Device initialization routines: put/get, init/exit
1236 ****************************************************************************/
1237
1238/*
1239 *      i7core_put_all_devices  'put' all the devices that we have
1240 *                              reserved via 'get'
1241 */
1242static void i7core_put_devices(struct i7core_dev *i7core_dev)
1243{
1244        int i;
1245
1246        debugf0(__FILE__ ": %s()\n", __func__);
1247        for (i = 0; i < i7core_dev->n_devs; i++) {
1248                struct pci_dev *pdev = i7core_dev->pdev[i];
1249                if (!pdev)
1250                        continue;
1251                debugf0("Removing dev %02x:%02x.%d\n",
1252                        pdev->bus->number,
1253                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1254                pci_dev_put(pdev);
1255        }
1256}
1257
1258static void i7core_put_all_devices(void)
1259{
1260        struct i7core_dev *i7core_dev, *tmp;
1261
1262        list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1263                i7core_put_devices(i7core_dev);
1264                free_i7core_dev(i7core_dev);
1265        }
1266}
1267
1268static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1269{
1270        struct pci_dev *pdev = NULL;
1271        int i;
1272
1273        /*
1274         * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1275         * aren't announced by acpi. So, we need to use a legacy scan probing
1276         * to detect them
1277         */
1278        while (table && table->descr) {
1279                pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1280                if (unlikely(!pdev)) {
1281                        for (i = 0; i < MAX_SOCKET_BUSES; i++)
1282                                pcibios_scan_specific_bus(255-i);
1283                }
1284                pci_dev_put(pdev);
1285                table++;
1286        }
1287}
1288
1289static unsigned i7core_pci_lastbus(void)
1290{
1291        int last_bus = 0, bus;
1292        struct pci_bus *b = NULL;
1293
1294        while ((b = pci_find_next_bus(b)) != NULL) {
1295                bus = b->number;
1296                debugf0("Found bus %d\n", bus);
1297                if (bus > last_bus)
1298                        last_bus = bus;
1299        }
1300
1301        debugf0("Last bus %d\n", last_bus);
1302
1303        return last_bus;
1304}
1305
1306/*
1307 *      i7core_get_all_devices  Find and perform 'get' operation on the MCH's
1308 *                      device/functions we want to reference for this driver
1309 *
1310 *                      Need to 'get' device 16 func 1 and func 2
1311 */
1312static int i7core_get_onedevice(struct pci_dev **prev,
1313                                const struct pci_id_table *table,
1314                                const unsigned devno,
1315                                const unsigned last_bus)
1316{
1317        struct i7core_dev *i7core_dev;
1318        const struct pci_id_descr *dev_descr = &table->descr[devno];
1319
1320        struct pci_dev *pdev = NULL;
1321        u8 bus = 0;
1322        u8 socket = 0;
1323
1324        pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1325                              dev_descr->dev_id, *prev);
1326
1327        if (!pdev) {
1328                if (*prev) {
1329                        *prev = pdev;
1330                        return 0;
1331                }
1332
1333                if (dev_descr->optional)
1334                        return 0;
1335
1336                if (devno == 0)
1337                        return -ENODEV;
1338
1339                i7core_printk(KERN_INFO,
1340                        "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1341                        dev_descr->dev, dev_descr->func,
1342                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1343
1344                /* End of list, leave */
1345                return -ENODEV;
1346        }
1347        bus = pdev->bus->number;
1348
1349        socket = last_bus - bus;
1350
1351        i7core_dev = get_i7core_dev(socket);
1352        if (!i7core_dev) {
1353                i7core_dev = alloc_i7core_dev(socket, table);
1354                if (!i7core_dev) {
1355                        pci_dev_put(pdev);
1356                        return -ENOMEM;
1357                }
1358        }
1359
1360        if (i7core_dev->pdev[devno]) {
1361                i7core_printk(KERN_ERR,
1362                        "Duplicated device for "
1363                        "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1364                        bus, dev_descr->dev, dev_descr->func,
1365                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1366                pci_dev_put(pdev);
1367                return -ENODEV;
1368        }
1369
1370        i7core_dev->pdev[devno] = pdev;
1371
1372        /* Sanity check */
1373        if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1374                        PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1375                i7core_printk(KERN_ERR,
1376                        "Device PCI ID %04x:%04x "
1377                        "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1378                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1379                        bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1380                        bus, dev_descr->dev, dev_descr->func);
1381                return -ENODEV;
1382        }
1383
1384        /* Be sure that the device is enabled */
1385        if (unlikely(pci_enable_device(pdev) < 0)) {
1386                i7core_printk(KERN_ERR,
1387                        "Couldn't enable "
1388                        "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1389                        bus, dev_descr->dev, dev_descr->func,
1390                        PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1391                return -ENODEV;
1392        }
1393
1394        debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1395                socket, bus, dev_descr->dev,
1396                dev_descr->func,
1397                PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1398
1399        /*
1400         * As stated on drivers/pci/search.c, the reference count for
1401         * @from is always decremented if it is not %NULL. So, as we need
1402         * to get all devices up to null, we need to do a get for the device
1403         */
1404        pci_dev_get(pdev);
1405
1406        *prev = pdev;
1407
1408        return 0;
1409}
1410
1411static int i7core_get_all_devices(void)
1412{
1413        int i, rc, last_bus;
1414        struct pci_dev *pdev = NULL;
1415        const struct pci_id_table *table = pci_dev_table;
1416
1417        last_bus = i7core_pci_lastbus();
1418
1419        while (table && table->descr) {
1420                for (i = 0; i < table->n_devs; i++) {
1421                        pdev = NULL;
1422                        do {
1423                                rc = i7core_get_onedevice(&pdev, table, i,
1424                                                          last_bus);
1425                                if (rc < 0) {
1426                                        if (i == 0) {
1427                                                i = table->n_devs;
1428                                                break;
1429                                        }
1430                                        i7core_put_all_devices();
1431                                        return -ENODEV;
1432                                }
1433                        } while (pdev);
1434                }
1435                table++;
1436        }
1437
1438        return 0;
1439}
1440
1441static int mci_bind_devs(struct mem_ctl_info *mci,
1442                         struct i7core_dev *i7core_dev)
1443{
1444        struct i7core_pvt *pvt = mci->pvt_info;
1445        struct pci_dev *pdev;
1446        int i, func, slot;
1447
1448        pvt->is_registered = 0;
1449        for (i = 0; i < i7core_dev->n_devs; i++) {
1450                pdev = i7core_dev->pdev[i];
1451                if (!pdev)
1452                        continue;
1453
1454                func = PCI_FUNC(pdev->devfn);
1455                slot = PCI_SLOT(pdev->devfn);
1456                if (slot == 3) {
1457                        if (unlikely(func > MAX_MCR_FUNC))
1458                                goto error;
1459                        pvt->pci_mcr[func] = pdev;
1460                } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1461                        if (unlikely(func > MAX_CHAN_FUNC))
1462                                goto error;
1463                        pvt->pci_ch[slot - 4][func] = pdev;
1464                } else if (!slot && !func)
1465                        pvt->pci_noncore = pdev;
1466                else
1467                        goto error;
1468
1469                debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1470                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1471                        pdev, i7core_dev->socket);
1472
1473                if (PCI_SLOT(pdev->devfn) == 3 &&
1474                        PCI_FUNC(pdev->devfn) == 2)
1475                        pvt->is_registered = 1;
1476        }
1477
1478        return 0;
1479
1480error:
1481        i7core_printk(KERN_ERR, "Device %d, function %d "
1482                      "is out of the expected range\n",
1483                      slot, func);
1484        return -EINVAL;
1485}
1486
1487/****************************************************************************
1488                        Error check routines
1489 ****************************************************************************/
1490static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1491                                      const int chan,
1492                                      const int dimm,
1493                                      const int add)
1494{
1495        char *msg;
1496        struct i7core_pvt *pvt = mci->pvt_info;
1497        int row = pvt->csrow_map[chan][dimm], i;
1498
1499        for (i = 0; i < add; i++) {
1500                msg = kasprintf(GFP_KERNEL, "Corrected error "
1501                                "(Socket=%d channel=%d dimm=%d)",
1502                                pvt->i7core_dev->socket, chan, dimm);
1503
1504                edac_mc_handle_fbd_ce(mci, row, 0, msg);
1505                kfree (msg);
1506        }
1507}
1508
1509static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1510                                         const int chan,
1511                                         const int new0,
1512                                         const int new1,
1513                                         const int new2)
1514{
1515        struct i7core_pvt *pvt = mci->pvt_info;
1516        int add0 = 0, add1 = 0, add2 = 0;
1517        /* Updates CE counters if it is not the first time here */
1518        if (pvt->ce_count_available) {
1519                /* Updates CE counters */
1520
1521                add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1522                add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1523                add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1524
1525                if (add2 < 0)
1526                        add2 += 0x7fff;
1527                pvt->rdimm_ce_count[chan][2] += add2;
1528
1529                if (add1 < 0)
1530                        add1 += 0x7fff;
1531                pvt->rdimm_ce_count[chan][1] += add1;
1532
1533                if (add0 < 0)
1534                        add0 += 0x7fff;
1535                pvt->rdimm_ce_count[chan][0] += add0;
1536        } else
1537                pvt->ce_count_available = 1;
1538
1539        /* Store the new values */
1540        pvt->rdimm_last_ce_count[chan][2] = new2;
1541        pvt->rdimm_last_ce_count[chan][1] = new1;
1542        pvt->rdimm_last_ce_count[chan][0] = new0;
1543
1544        /*updated the edac core */
1545        if (add0 != 0)
1546                i7core_rdimm_update_csrow(mci, chan, 0, add0);
1547        if (add1 != 0)
1548                i7core_rdimm_update_csrow(mci, chan, 1, add1);
1549        if (add2 != 0)
1550                i7core_rdimm_update_csrow(mci, chan, 2, add2);
1551
1552}
1553
1554static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1555{
1556        struct i7core_pvt *pvt = mci->pvt_info;
1557        u32 rcv[3][2];
1558        int i, new0, new1, new2;
1559
1560        /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1561        pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1562                                                                &rcv[0][0]);
1563        pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1564                                                                &rcv[0][1]);
1565        pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1566                                                                &rcv[1][0]);
1567        pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1568                                                                &rcv[1][1]);
1569        pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1570                                                                &rcv[2][0]);
1571        pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1572                                                                &rcv[2][1]);
1573        for (i = 0 ; i < 3; i++) {
1574                debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1575                        (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1576                /*if the channel has 3 dimms*/
1577                if (pvt->channel[i].dimms > 2) {
1578                        new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1579                        new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1580                        new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1581                } else {
1582                        new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1583                                        DIMM_BOT_COR_ERR(rcv[i][0]);
1584                        new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1585                                        DIMM_BOT_COR_ERR(rcv[i][1]);
1586                        new2 = 0;
1587                }
1588
1589                i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1590        }
1591}
1592
1593/* This function is based on the device 3 function 4 registers as described on:
1594 * Intel Xeon Processor 5500 Series Datasheet Volume 2
1595 *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1596 * also available at:
1597 *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1598 */
1599static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1600{
1601        struct i7core_pvt *pvt = mci->pvt_info;
1602        u32 rcv1, rcv0;
1603        int new0, new1, new2;
1604
1605        if (!pvt->pci_mcr[4]) {
1606                debugf0("%s MCR registers not found\n", __func__);
1607                return;
1608        }
1609
1610        /* Corrected test errors */
1611        pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1612        pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1613
1614        /* Store the new values */
1615        new2 = DIMM2_COR_ERR(rcv1);
1616        new1 = DIMM1_COR_ERR(rcv0);
1617        new0 = DIMM0_COR_ERR(rcv0);
1618
1619        /* Updates CE counters if it is not the first time here */
1620        if (pvt->ce_count_available) {
1621                /* Updates CE counters */
1622                int add0, add1, add2;
1623
1624                add2 = new2 - pvt->udimm_last_ce_count[2];
1625                add1 = new1 - pvt->udimm_last_ce_count[1];
1626                add0 = new0 - pvt->udimm_last_ce_count[0];
1627
1628                if (add2 < 0)
1629                        add2 += 0x7fff;
1630                pvt->udimm_ce_count[2] += add2;
1631
1632                if (add1 < 0)
1633                        add1 += 0x7fff;
1634                pvt->udimm_ce_count[1] += add1;
1635
1636                if (add0 < 0)
1637                        add0 += 0x7fff;
1638                pvt->udimm_ce_count[0] += add0;
1639
1640                if (add0 | add1 | add2)
1641                        i7core_printk(KERN_ERR, "New Corrected error(s): "
1642                                      "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1643                                      add0, add1, add2);
1644        } else
1645                pvt->ce_count_available = 1;
1646
1647        /* Store the new values */
1648        pvt->udimm_last_ce_count[2] = new2;
1649        pvt->udimm_last_ce_count[1] = new1;
1650        pvt->udimm_last_ce_count[0] = new0;
1651}
1652
1653/*
1654 * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1655 * Architectures Software Developer\xE2\x80\x99s Manual Volume 3B.
1656 * Nehalem are defined as family 0x06, model 0x1a
1657 *
1658 * The MCA registers used here are the following ones:
1659 *     struct mce field MCA Register
1660 *     m->status        MSR_IA32_MC8_STATUS
1661 *     m->addr          MSR_IA32_MC8_ADDR
1662 *     m->misc          MSR_IA32_MC8_MISC
1663 * In the case of Nehalem, the error information is masked at .status and .misc
1664 * fields
1665 */
1666static void i7core_mce_output_error(struct mem_ctl_info *mci,
1667                                    const struct mce *m)
1668{
1669        struct i7core_pvt *pvt = mci->pvt_info;
1670        char *type, *optype, *err, *msg;
1671        unsigned long error = m->status & 0x1ff0000l;
1672        u32 optypenum = (m->status >> 4) & 0x07;
1673        u32 core_err_cnt = (m->status >> 38) && 0x7fff;
1674        u32 dimm = (m->misc >> 16) & 0x3;
1675        u32 channel = (m->misc >> 18) & 0x3;
1676        u32 syndrome = m->misc >> 32;
1677        u32 errnum = find_first_bit(&error, 32);
1678        int csrow;
1679
1680        if (m->mcgstatus & 1)
1681                type = "FATAL";
1682        else
1683                type = "NON_FATAL";
1684
1685        switch (optypenum) {
1686        case 0:
1687                optype = "generic undef request";
1688                break;
1689        case 1:
1690                optype = "read error";
1691                break;
1692        case 2:
1693                optype = "write error";
1694                break;
1695        case 3:
1696                optype = "addr/cmd error";
1697                break;
1698        case 4:
1699                optype = "scrubbing error";
1700                break;
1701        default:
1702                optype = "reserved";
1703                break;
1704        }
1705
1706        switch (errnum) {
1707        case 16:
1708                err = "read ECC error";
1709                break;
1710        case 17:
1711                err = "RAS ECC error";
1712                break;
1713        case 18:
1714                err = "write parity error";
1715                break;
1716        case 19:
1717                err = "redundacy loss";
1718                break;
1719        case 20:
1720                err = "reserved";
1721                break;
1722        case 21:
1723                err = "memory range error";
1724                break;
1725        case 22:
1726                err = "RTID out of range";
1727                break;
1728        case 23:
1729                err = "address parity error";
1730                break;
1731        case 24:
1732                err = "byte enable parity error";
1733                break;
1734        default:
1735                err = "unknown";
1736        }
1737
1738        /* FIXME: should convert addr into bank and rank information */
1739        msg = kasprintf(GFP_ATOMIC,
1740                "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1741                "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1742                type, (long long) m->addr, m->cpu, dimm, channel,
1743                syndrome, core_err_cnt, (long long)m->status,
1744                (long long)m->misc, optype, err);
1745
1746        debugf0("%s", msg);
1747
1748        csrow = pvt->csrow_map[channel][dimm];
1749
1750        /* Call the helper to output message */
1751        if (m->mcgstatus & 1)
1752                edac_mc_handle_fbd_ue(mci, csrow, 0,
1753                                0 /* FIXME: should be channel here */, msg);
1754        else if (!pvt->is_registered)
1755                edac_mc_handle_fbd_ce(mci, csrow,
1756                                0 /* FIXME: should be channel here */, msg);
1757
1758        kfree(msg);
1759}
1760
1761/*
1762 *      i7core_check_error      Retrieve and process errors reported by the
1763 *                              hardware. Called by the Core module.
1764 */
1765static void i7core_check_error(struct mem_ctl_info *mci)
1766{
1767        struct i7core_pvt *pvt = mci->pvt_info;
1768        int i;
1769        unsigned count = 0;
1770        struct mce *m;
1771
1772        /*
1773         * MCE first step: Copy all mce errors into a temporary buffer
1774         * We use a double buffering here, to reduce the risk of
1775         * loosing an error.
1776         */
1777        smp_rmb();
1778        count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1779                % MCE_LOG_LEN;
1780        if (!count)
1781                goto check_ce_error;
1782
1783        m = pvt->mce_outentry;
1784        if (pvt->mce_in + count > MCE_LOG_LEN) {
1785                unsigned l = MCE_LOG_LEN - pvt->mce_in;
1786
1787                memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1788                smp_wmb();
1789                pvt->mce_in = 0;
1790                count -= l;
1791                m += l;
1792        }
1793        memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1794        smp_wmb();
1795        pvt->mce_in += count;
1796
1797        smp_rmb();
1798        if (pvt->mce_overrun) {
1799                i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1800                              pvt->mce_overrun);
1801                smp_wmb();
1802                pvt->mce_overrun = 0;
1803        }
1804
1805        /*
1806         * MCE second step: parse errors and display
1807         */
1808        for (i = 0; i < count; i++)
1809                i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1810
1811        /*
1812         * Now, let's increment CE error counts
1813         */
1814check_ce_error:
1815        if (!pvt->is_registered)
1816                i7core_udimm_check_mc_ecc_err(mci);
1817        else
1818                i7core_rdimm_check_mc_ecc_err(mci);
1819}
1820
1821/*
1822 * i7core_mce_check_error       Replicates mcelog routine to get errors
1823 *                              This routine simply queues mcelog errors, and
1824 *                              return. The error itself should be handled later
1825 *                              by i7core_check_error.
1826 * WARNING: As this routine should be called at NMI time, extra care should
1827 * be taken to avoid deadlocks, and to be as fast as possible.
1828 */
1829static int i7core_mce_check_error(void *priv, struct mce *mce)
1830{
1831        struct mem_ctl_info *mci = priv;
1832        struct i7core_pvt *pvt = mci->pvt_info;
1833
1834        /*
1835         * Just let mcelog handle it if the error is
1836         * outside the memory controller
1837         */
1838        if (((mce->status & 0xffff) >> 7) != 1)
1839                return 0;
1840
1841        /* Bank 8 registers are the only ones that we know how to handle */
1842        if (mce->bank != 8)
1843                return 0;
1844
1845#ifdef CONFIG_SMP
1846        /* Only handle if it is the right mc controller */
1847        if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket)
1848                return 0;
1849#endif
1850
1851        smp_rmb();
1852        if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1853                smp_wmb();
1854                pvt->mce_overrun++;
1855                return 0;
1856        }
1857
1858        /* Copy memory error at the ringbuffer */
1859        memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1860        smp_wmb();
1861        pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1862
1863        /* Handle fatal errors immediately */
1864        if (mce->mcgstatus & 1)
1865                i7core_check_error(mci);
1866
1867        /* Advise mcelog that the errors were handled */
1868        return 1;
1869}
1870
1871static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
1872{
1873        pvt->i7core_pci = edac_pci_create_generic_ctl(
1874                                                &pvt->i7core_dev->pdev[0]->dev,
1875                                                EDAC_MOD_STR);
1876        if (unlikely(!pvt->i7core_pci))
1877                pr_warn("Unable to setup PCI error report via EDAC\n");
1878}
1879
1880static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
1881{
1882        if (likely(pvt->i7core_pci))
1883                edac_pci_release_generic_ctl(pvt->i7core_pci);
1884        else
1885                i7core_printk(KERN_ERR,
1886                                "Couldn't find mem_ctl_info for socket %d\n",
1887                                pvt->i7core_dev->socket);
1888        pvt->i7core_pci = NULL;
1889}
1890
1891static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
1892{
1893        struct mem_ctl_info *mci = i7core_dev->mci;
1894        struct i7core_pvt *pvt;
1895
1896        if (unlikely(!mci || !mci->pvt_info)) {
1897                debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
1898                        __func__, &i7core_dev->pdev[0]->dev);
1899
1900                i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
1901                return;
1902        }
1903
1904        pvt = mci->pvt_info;
1905
1906        debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
1907                __func__, mci, &i7core_dev->pdev[0]->dev);
1908
1909        /* Disable MCE NMI handler */
1910        edac_mce_unregister(&pvt->edac_mce);
1911
1912        /* Disable EDAC polling */
1913        i7core_pci_ctl_release(pvt);
1914
1915        /* Remove MC sysfs nodes */
1916        edac_mc_del_mc(mci->dev);
1917
1918        debugf1("%s: free mci struct\n", mci->ctl_name);
1919        kfree(mci->ctl_name);
1920        edac_mc_free(mci);
1921        i7core_dev->mci = NULL;
1922}
1923
1924static int i7core_register_mci(struct i7core_dev *i7core_dev)
1925{
1926        struct mem_ctl_info *mci;
1927        struct i7core_pvt *pvt;
1928        int rc, channels, csrows;
1929
1930        /* Check the number of active and not disabled channels */
1931        rc = i7core_get_active_channels(i7core_dev->socket, &channels, &csrows);
1932        if (unlikely(rc < 0))
1933                return rc;
1934
1935        /* allocate a new MC control structure */
1936        mci = edac_mc_alloc(sizeof(*pvt), csrows, channels, i7core_dev->socket);
1937        if (unlikely(!mci))
1938                return -ENOMEM;
1939
1940        debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
1941                __func__, mci, &i7core_dev->pdev[0]->dev);
1942
1943        pvt = mci->pvt_info;
1944        memset(pvt, 0, sizeof(*pvt));
1945
1946        /* Associates i7core_dev and mci for future usage */
1947        pvt->i7core_dev = i7core_dev;
1948        i7core_dev->mci = mci;
1949
1950        /*
1951         * FIXME: how to handle RDDR3 at MCI level? It is possible to have
1952         * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
1953         * memory channels
1954         */
1955        mci->mtype_cap = MEM_FLAG_DDR3;
1956        mci->edac_ctl_cap = EDAC_FLAG_NONE;
1957        mci->edac_cap = EDAC_FLAG_NONE;
1958        mci->mod_name = "i7core_edac.c";
1959        mci->mod_ver = I7CORE_REVISION;
1960        mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
1961                                  i7core_dev->socket);
1962        mci->dev_name = pci_name(i7core_dev->pdev[0]);
1963        mci->ctl_page_to_phys = NULL;
1964
1965        /* Store pci devices at mci for faster access */
1966        rc = mci_bind_devs(mci, i7core_dev);
1967        if (unlikely(rc < 0))
1968                goto fail0;
1969
1970        if (pvt->is_registered)
1971                mci->mc_driver_sysfs_attributes = i7core_sysfs_rdimm_attrs;
1972        else
1973                mci->mc_driver_sysfs_attributes = i7core_sysfs_udimm_attrs;
1974
1975        /* Get dimm basic config */
1976        get_dimm_config(mci);
1977        /* record ptr to the generic device */
1978        mci->dev = &i7core_dev->pdev[0]->dev;
1979        /* Set the function pointer to an actual operation function */
1980        mci->edac_check = i7core_check_error;
1981
1982        /* add this new MC control structure to EDAC's list of MCs */
1983        if (unlikely(edac_mc_add_mc(mci))) {
1984                debugf0("MC: " __FILE__
1985                        ": %s(): failed edac_mc_add_mc()\n", __func__);
1986                /* FIXME: perhaps some code should go here that disables error
1987                 * reporting if we just enabled it
1988                 */
1989
1990                rc = -EINVAL;
1991                goto fail0;
1992        }
1993
1994        /* Default error mask is any memory */
1995        pvt->inject.channel = 0;
1996        pvt->inject.dimm = -1;
1997        pvt->inject.rank = -1;
1998        pvt->inject.bank = -1;
1999        pvt->inject.page = -1;
2000        pvt->inject.col = -1;
2001
2002        /* allocating generic PCI control info */
2003        i7core_pci_ctl_create(pvt);
2004
2005        /* Registers on edac_mce in order to receive memory errors */
2006        pvt->edac_mce.priv = mci;
2007        pvt->edac_mce.check_error = i7core_mce_check_error;
2008        rc = edac_mce_register(&pvt->edac_mce);
2009        if (unlikely(rc < 0)) {
2010                debugf0("MC: " __FILE__
2011                        ": %s(): failed edac_mce_register()\n", __func__);
2012                goto fail1;
2013        }
2014
2015        return 0;
2016
2017fail1:
2018        i7core_pci_ctl_release(pvt);
2019        edac_mc_del_mc(mci->dev);
2020fail0:
2021        kfree(mci->ctl_name);
2022        edac_mc_free(mci);
2023        i7core_dev->mci = NULL;
2024        return rc;
2025}
2026
2027/*
2028 *      i7core_probe    Probe for ONE instance of device to see if it is
2029 *                      present.
2030 *      return:
2031 *              0 for FOUND a device
2032 *              < 0 for error code
2033 */
2034
2035static int __devinit i7core_probe(struct pci_dev *pdev,
2036                                  const struct pci_device_id *id)
2037{
2038        int rc;
2039        struct i7core_dev *i7core_dev;
2040
2041        /* get the pci devices we want to reserve for our use */
2042        mutex_lock(&i7core_edac_lock);
2043
2044        /*
2045         * All memory controllers are allocated at the first pass.
2046         */
2047        if (unlikely(probed >= 1)) {
2048                mutex_unlock(&i7core_edac_lock);
2049                return -ENODEV;
2050        }
2051        probed++;
2052
2053        rc = i7core_get_all_devices();
2054        if (unlikely(rc < 0))
2055                goto fail0;
2056
2057        list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2058                rc = i7core_register_mci(i7core_dev);
2059                if (unlikely(rc < 0))
2060                        goto fail1;
2061        }
2062
2063        i7core_printk(KERN_INFO, "Driver loaded.\n");
2064
2065        mutex_unlock(&i7core_edac_lock);
2066        return 0;
2067
2068fail1:
2069        list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2070                i7core_unregister_mci(i7core_dev);
2071
2072        i7core_put_all_devices();
2073fail0:
2074        mutex_unlock(&i7core_edac_lock);
2075        return rc;
2076}
2077
2078/*
2079 *      i7core_remove   destructor for one instance of device
2080 *
2081 */
2082static void __devexit i7core_remove(struct pci_dev *pdev)
2083{
2084        struct i7core_dev *i7core_dev;
2085
2086        debugf0(__FILE__ ": %s()\n", __func__);
2087
2088        /*
2089         * we have a trouble here: pdev value for removal will be wrong, since
2090         * it will point to the X58 register used to detect that the machine
2091         * is a Nehalem or upper design. However, due to the way several PCI
2092         * devices are grouped together to provide MC functionality, we need
2093         * to use a different method for releasing the devices
2094         */
2095
2096        mutex_lock(&i7core_edac_lock);
2097
2098        if (unlikely(!probed)) {
2099                mutex_unlock(&i7core_edac_lock);
2100                return;
2101        }
2102
2103        list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2104                i7core_unregister_mci(i7core_dev);
2105
2106        /* Release PCI resources */
2107        i7core_put_all_devices();
2108
2109        probed--;
2110
2111        mutex_unlock(&i7core_edac_lock);
2112}
2113
2114MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2115
2116/*
2117 *      i7core_driver   pci_driver structure for this module
2118 *
2119 */
2120static struct pci_driver i7core_driver = {
2121        .name     = "i7core_edac",
2122        .probe    = i7core_probe,
2123        .remove   = __devexit_p(i7core_remove),
2124        .id_table = i7core_pci_tbl,
2125};
2126
2127/*
2128 *      i7core_init             Module entry function
2129 *                      Try to initialize this module for its devices
2130 */
2131static int __init i7core_init(void)
2132{
2133        int pci_rc;
2134
2135        debugf2("MC: " __FILE__ ": %s()\n", __func__);
2136
2137        /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2138        opstate_init();
2139
2140        if (use_pci_fixup)
2141                i7core_xeon_pci_fixup(pci_dev_table);
2142
2143        pci_rc = pci_register_driver(&i7core_driver);
2144
2145        if (pci_rc >= 0)
2146                return 0;
2147
2148        i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2149                      pci_rc);
2150
2151        return pci_rc;
2152}
2153
2154/*
2155 *      i7core_exit()   Module exit function
2156 *                      Unregister the driver
2157 */
2158static void __exit i7core_exit(void)
2159{
2160        debugf2("MC: " __FILE__ ": %s()\n", __func__);
2161        pci_unregister_driver(&i7core_driver);
2162}
2163
2164module_init(i7core_init);
2165module_exit(i7core_exit);
2166
2167MODULE_LICENSE("GPL");
2168MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2169MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2170MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2171                   I7CORE_REVISION);
2172
2173module_param(edac_op_state, int, 0444);
2174MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
2175