linux/drivers/edac/amd64_edac.c
<<
>>
Prefs
   1#include "amd64_edac.h"
   2#include <asm/amd_nb.h>
   3
   4static struct edac_pci_ctl_info *amd64_ctl_pci;
   5
   6static int report_gart_errors;
   7module_param(report_gart_errors, int, 0644);
   8
   9/*
  10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
  11 * cleared to prevent re-enabling the hardware by this driver.
  12 */
  13static int ecc_enable_override;
  14module_param(ecc_enable_override, int, 0644);
  15
  16static struct msr __percpu *msrs;
  17
  18/*
  19 * count successfully initialized driver instances for setup_pci_device()
  20 */
  21static atomic_t drv_instances = ATOMIC_INIT(0);
  22
  23/* Per-node driver instances */
  24static struct mem_ctl_info **mcis;
  25static struct ecc_settings **ecc_stngs;
  26
  27/*
  28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
  29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
  30 * or higher value'.
  31 *
  32 *FIXME: Produce a better mapping/linearisation.
  33 */
  34static const struct scrubrate {
  35       u32 scrubval;           /* bit pattern for scrub rate */
  36       u32 bandwidth;          /* bandwidth consumed (bytes/sec) */
  37} scrubrates[] = {
  38        { 0x01, 1600000000UL},
  39        { 0x02, 800000000UL},
  40        { 0x03, 400000000UL},
  41        { 0x04, 200000000UL},
  42        { 0x05, 100000000UL},
  43        { 0x06, 50000000UL},
  44        { 0x07, 25000000UL},
  45        { 0x08, 12284069UL},
  46        { 0x09, 6274509UL},
  47        { 0x0A, 3121951UL},
  48        { 0x0B, 1560975UL},
  49        { 0x0C, 781440UL},
  50        { 0x0D, 390720UL},
  51        { 0x0E, 195300UL},
  52        { 0x0F, 97650UL},
  53        { 0x10, 48854UL},
  54        { 0x11, 24427UL},
  55        { 0x12, 12213UL},
  56        { 0x13, 6101UL},
  57        { 0x14, 3051UL},
  58        { 0x15, 1523UL},
  59        { 0x16, 761UL},
  60        { 0x00, 0UL},        /* scrubbing off */
  61};
  62
  63int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
  64                               u32 *val, const char *func)
  65{
  66        int err = 0;
  67
  68        err = pci_read_config_dword(pdev, offset, val);
  69        if (err)
  70                amd64_warn("%s: error reading F%dx%03x.\n",
  71                           func, PCI_FUNC(pdev->devfn), offset);
  72
  73        return err;
  74}
  75
  76int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
  77                                u32 val, const char *func)
  78{
  79        int err = 0;
  80
  81        err = pci_write_config_dword(pdev, offset, val);
  82        if (err)
  83                amd64_warn("%s: error writing to F%dx%03x.\n",
  84                           func, PCI_FUNC(pdev->devfn), offset);
  85
  86        return err;
  87}
  88
  89/*
  90 *
  91 * Depending on the family, F2 DCT reads need special handling:
  92 *
  93 * K8: has a single DCT only
  94 *
  95 * F10h: each DCT has its own set of regs
  96 *      DCT0 -> F2x040..
  97 *      DCT1 -> F2x140..
  98 *
  99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
 100 *
 101 */
 102static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
 103                               const char *func)
 104{
 105        if (addr >= 0x100)
 106                return -EINVAL;
 107
 108        return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
 109}
 110
 111static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
 112                                 const char *func)
 113{
 114        return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
 115}
 116
 117/*
 118 * Select DCT to which PCI cfg accesses are routed
 119 */
 120static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
 121{
 122        u32 reg = 0;
 123
 124        amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
 125        reg &= 0xfffffffe;
 126        reg |= dct;
 127        amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
 128}
 129
 130static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
 131                                 const char *func)
 132{
 133        u8 dct  = 0;
 134
 135        if (addr >= 0x140 && addr <= 0x1a0) {
 136                dct   = 1;
 137                addr -= 0x100;
 138        }
 139
 140        f15h_select_dct(pvt, dct);
 141
 142        return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
 143}
 144
 145/*
 146 * Memory scrubber control interface. For K8, memory scrubbing is handled by
 147 * hardware and can involve L2 cache, dcache as well as the main memory. With
 148 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
 149 * functionality.
 150 *
 151 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
 152 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
 153 * bytes/sec for the setting.
 154 *
 155 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
 156 * other archs, we might not have access to the caches directly.
 157 */
 158
 159/*
 160 * scan the scrub rate mapping table for a close or matching bandwidth value to
 161 * issue. If requested is too big, then use last maximum value found.
 162 */
 163static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
 164{
 165        u32 scrubval;
 166        int i;
 167
 168        /*
 169         * map the configured rate (new_bw) to a value specific to the AMD64
 170         * memory controller and apply to register. Search for the first
 171         * bandwidth entry that is greater or equal than the setting requested
 172         * and program that. If at last entry, turn off DRAM scrubbing.
 173         *
 174         * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
 175         * by falling back to the last element in scrubrates[].
 176         */
 177        for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
 178                /*
 179                 * skip scrub rates which aren't recommended
 180                 * (see F10 BKDG, F3x58)
 181                 */
 182                if (scrubrates[i].scrubval < min_rate)
 183                        continue;
 184
 185                if (scrubrates[i].bandwidth <= new_bw)
 186                        break;
 187        }
 188
 189        scrubval = scrubrates[i].scrubval;
 190
 191        pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
 192
 193        if (scrubval)
 194                return scrubrates[i].bandwidth;
 195
 196        return 0;
 197}
 198
 199static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
 200{
 201        struct amd64_pvt *pvt = mci->pvt_info;
 202        u32 min_scrubrate = 0x5;
 203
 204        if (boot_cpu_data.x86 == 0xf)
 205                min_scrubrate = 0x0;
 206
 207        /* F15h Erratum #505 */
 208        if (boot_cpu_data.x86 == 0x15)
 209                f15h_select_dct(pvt, 0);
 210
 211        return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
 212}
 213
 214static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
 215{
 216        struct amd64_pvt *pvt = mci->pvt_info;
 217        u32 scrubval = 0;
 218        int i, retval = -EINVAL;
 219
 220        /* F15h Erratum #505 */
 221        if (boot_cpu_data.x86 == 0x15)
 222                f15h_select_dct(pvt, 0);
 223
 224        amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
 225
 226        scrubval = scrubval & 0x001F;
 227
 228        for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
 229                if (scrubrates[i].scrubval == scrubval) {
 230                        retval = scrubrates[i].bandwidth;
 231                        break;
 232                }
 233        }
 234        return retval;
 235}
 236
 237/*
 238 * returns true if the SysAddr given by sys_addr matches the
 239 * DRAM base/limit associated with node_id
 240 */
 241static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
 242                                   u8 nid)
 243{
 244        u64 addr;
 245
 246        /* The K8 treats this as a 40-bit value.  However, bits 63-40 will be
 247         * all ones if the most significant implemented address bit is 1.
 248         * Here we discard bits 63-40.  See section 3.4.2 of AMD publication
 249         * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
 250         * Application Programming.
 251         */
 252        addr = sys_addr & 0x000000ffffffffffull;
 253
 254        return ((addr >= get_dram_base(pvt, nid)) &&
 255                (addr <= get_dram_limit(pvt, nid)));
 256}
 257
 258/*
 259 * Attempt to map a SysAddr to a node. On success, return a pointer to the
 260 * mem_ctl_info structure for the node that the SysAddr maps to.
 261 *
 262 * On failure, return NULL.
 263 */
 264static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
 265                                                u64 sys_addr)
 266{
 267        struct amd64_pvt *pvt;
 268        u8 node_id;
 269        u32 intlv_en, bits;
 270
 271        /*
 272         * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
 273         * 3.4.4.2) registers to map the SysAddr to a node ID.
 274         */
 275        pvt = mci->pvt_info;
 276
 277        /*
 278         * The value of this field should be the same for all DRAM Base
 279         * registers.  Therefore we arbitrarily choose to read it from the
 280         * register for node 0.
 281         */
 282        intlv_en = dram_intlv_en(pvt, 0);
 283
 284        if (intlv_en == 0) {
 285                for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
 286                        if (amd64_base_limit_match(pvt, sys_addr, node_id))
 287                                goto found;
 288                }
 289                goto err_no_match;
 290        }
 291
 292        if (unlikely((intlv_en != 0x01) &&
 293                     (intlv_en != 0x03) &&
 294                     (intlv_en != 0x07))) {
 295                amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
 296                return NULL;
 297        }
 298
 299        bits = (((u32) sys_addr) >> 12) & intlv_en;
 300
 301        for (node_id = 0; ; ) {
 302                if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
 303                        break;  /* intlv_sel field matches */
 304
 305                if (++node_id >= DRAM_RANGES)
 306                        goto err_no_match;
 307        }
 308
 309        /* sanity test for sys_addr */
 310        if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
 311                amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
 312                           "range for node %d with node interleaving enabled.\n",
 313                           __func__, sys_addr, node_id);
 314                return NULL;
 315        }
 316
 317found:
 318        return edac_mc_find((int)node_id);
 319
 320err_no_match:
 321        edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
 322                 (unsigned long)sys_addr);
 323
 324        return NULL;
 325}
 326
 327/*
 328 * compute the CS base address of the @csrow on the DRAM controller @dct.
 329 * For details see F2x[5C:40] in the processor's BKDG
 330 */
 331static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
 332                                 u64 *base, u64 *mask)
 333{
 334        u64 csbase, csmask, base_bits, mask_bits;
 335        u8 addr_shift;
 336
 337        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
 338                csbase          = pvt->csels[dct].csbases[csrow];
 339                csmask          = pvt->csels[dct].csmasks[csrow];
 340                base_bits       = GENMASK(21, 31) | GENMASK(9, 15);
 341                mask_bits       = GENMASK(21, 29) | GENMASK(9, 15);
 342                addr_shift      = 4;
 343        } else {
 344                csbase          = pvt->csels[dct].csbases[csrow];
 345                csmask          = pvt->csels[dct].csmasks[csrow >> 1];
 346                addr_shift      = 8;
 347
 348                if (boot_cpu_data.x86 == 0x15)
 349                        base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
 350                else
 351                        base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
 352        }
 353
 354        *base  = (csbase & base_bits) << addr_shift;
 355
 356        *mask  = ~0ULL;
 357        /* poke holes for the csmask */
 358        *mask &= ~(mask_bits << addr_shift);
 359        /* OR them in */
 360        *mask |= (csmask & mask_bits) << addr_shift;
 361}
 362
 363#define for_each_chip_select(i, dct, pvt) \
 364        for (i = 0; i < pvt->csels[dct].b_cnt; i++)
 365
 366#define chip_select_base(i, dct, pvt) \
 367        pvt->csels[dct].csbases[i]
 368
 369#define for_each_chip_select_mask(i, dct, pvt) \
 370        for (i = 0; i < pvt->csels[dct].m_cnt; i++)
 371
 372/*
 373 * @input_addr is an InputAddr associated with the node given by mci. Return the
 374 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
 375 */
 376static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
 377{
 378        struct amd64_pvt *pvt;
 379        int csrow;
 380        u64 base, mask;
 381
 382        pvt = mci->pvt_info;
 383
 384        for_each_chip_select(csrow, 0, pvt) {
 385                if (!csrow_enabled(csrow, 0, pvt))
 386                        continue;
 387
 388                get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
 389
 390                mask = ~mask;
 391
 392                if ((input_addr & mask) == (base & mask)) {
 393                        edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
 394                                 (unsigned long)input_addr, csrow,
 395                                 pvt->mc_node_id);
 396
 397                        return csrow;
 398                }
 399        }
 400        edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
 401                 (unsigned long)input_addr, pvt->mc_node_id);
 402
 403        return -1;
 404}
 405
 406/*
 407 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
 408 * for the node represented by mci. Info is passed back in *hole_base,
 409 * *hole_offset, and *hole_size.  Function returns 0 if info is valid or 1 if
 410 * info is invalid. Info may be invalid for either of the following reasons:
 411 *
 412 * - The revision of the node is not E or greater.  In this case, the DRAM Hole
 413 *   Address Register does not exist.
 414 *
 415 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
 416 *   indicating that its contents are not valid.
 417 *
 418 * The values passed back in *hole_base, *hole_offset, and *hole_size are
 419 * complete 32-bit values despite the fact that the bitfields in the DHAR
 420 * only represent bits 31-24 of the base and offset values.
 421 */
 422int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
 423                             u64 *hole_offset, u64 *hole_size)
 424{
 425        struct amd64_pvt *pvt = mci->pvt_info;
 426
 427        /* only revE and later have the DRAM Hole Address Register */
 428        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
 429                edac_dbg(1, "  revision %d for node %d does not support DHAR\n",
 430                         pvt->ext_model, pvt->mc_node_id);
 431                return 1;
 432        }
 433
 434        /* valid for Fam10h and above */
 435        if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
 436                edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this system\n");
 437                return 1;
 438        }
 439
 440        if (!dhar_valid(pvt)) {
 441                edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this node %d\n",
 442                         pvt->mc_node_id);
 443                return 1;
 444        }
 445
 446        /* This node has Memory Hoisting */
 447
 448        /* +------------------+--------------------+--------------------+-----
 449         * | memory           | DRAM hole          | relocated          |
 450         * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
 451         * |                  |                    | DRAM hole          |
 452         * |                  |                    | [0x100000000,      |
 453         * |                  |                    |  (0x100000000+     |
 454         * |                  |                    |   (0xffffffff-x))] |
 455         * +------------------+--------------------+--------------------+-----
 456         *
 457         * Above is a diagram of physical memory showing the DRAM hole and the
 458         * relocated addresses from the DRAM hole.  As shown, the DRAM hole
 459         * starts at address x (the base address) and extends through address
 460         * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
 461         * addresses in the hole so that they start at 0x100000000.
 462         */
 463
 464        *hole_base = dhar_base(pvt);
 465        *hole_size = (1ULL << 32) - *hole_base;
 466
 467        if (boot_cpu_data.x86 > 0xf)
 468                *hole_offset = f10_dhar_offset(pvt);
 469        else
 470                *hole_offset = k8_dhar_offset(pvt);
 471
 472        edac_dbg(1, "  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
 473                 pvt->mc_node_id, (unsigned long)*hole_base,
 474                 (unsigned long)*hole_offset, (unsigned long)*hole_size);
 475
 476        return 0;
 477}
 478EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
 479
 480/*
 481 * Return the DramAddr that the SysAddr given by @sys_addr maps to.  It is
 482 * assumed that sys_addr maps to the node given by mci.
 483 *
 484 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
 485 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
 486 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
 487 * then it is also involved in translating a SysAddr to a DramAddr. Sections
 488 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
 489 * These parts of the documentation are unclear. I interpret them as follows:
 490 *
 491 * When node n receives a SysAddr, it processes the SysAddr as follows:
 492 *
 493 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
 494 *    Limit registers for node n. If the SysAddr is not within the range
 495 *    specified by the base and limit values, then node n ignores the Sysaddr
 496 *    (since it does not map to node n). Otherwise continue to step 2 below.
 497 *
 498 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
 499 *    disabled so skip to step 3 below. Otherwise see if the SysAddr is within
 500 *    the range of relocated addresses (starting at 0x100000000) from the DRAM
 501 *    hole. If not, skip to step 3 below. Else get the value of the
 502 *    DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
 503 *    offset defined by this value from the SysAddr.
 504 *
 505 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
 506 *    Base register for node n. To obtain the DramAddr, subtract the base
 507 *    address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
 508 */
 509static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
 510{
 511        struct amd64_pvt *pvt = mci->pvt_info;
 512        u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
 513        int ret;
 514
 515        dram_base = get_dram_base(pvt, pvt->mc_node_id);
 516
 517        ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
 518                                      &hole_size);
 519        if (!ret) {
 520                if ((sys_addr >= (1ULL << 32)) &&
 521                    (sys_addr < ((1ULL << 32) + hole_size))) {
 522                        /* use DHAR to translate SysAddr to DramAddr */
 523                        dram_addr = sys_addr - hole_offset;
 524
 525                        edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
 526                                 (unsigned long)sys_addr,
 527                                 (unsigned long)dram_addr);
 528
 529                        return dram_addr;
 530                }
 531        }
 532
 533        /*
 534         * Translate the SysAddr to a DramAddr as shown near the start of
 535         * section 3.4.4 (p. 70).  Although sys_addr is a 64-bit value, the k8
 536         * only deals with 40-bit values.  Therefore we discard bits 63-40 of
 537         * sys_addr below.  If bit 39 of sys_addr is 1 then the bits we
 538         * discard are all 1s.  Otherwise the bits we discard are all 0s.  See
 539         * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
 540         * Programmer's Manual Volume 1 Application Programming.
 541         */
 542        dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
 543
 544        edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
 545                 (unsigned long)sys_addr, (unsigned long)dram_addr);
 546        return dram_addr;
 547}
 548
 549/*
 550 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
 551 * (section 3.4.4.1).  Return the number of bits from a SysAddr that are used
 552 * for node interleaving.
 553 */
 554static int num_node_interleave_bits(unsigned intlv_en)
 555{
 556        static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
 557        int n;
 558
 559        BUG_ON(intlv_en > 7);
 560        n = intlv_shift_table[intlv_en];
 561        return n;
 562}
 563
 564/* Translate the DramAddr given by @dram_addr to an InputAddr. */
 565static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
 566{
 567        struct amd64_pvt *pvt;
 568        int intlv_shift;
 569        u64 input_addr;
 570
 571        pvt = mci->pvt_info;
 572
 573        /*
 574         * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
 575         * concerning translating a DramAddr to an InputAddr.
 576         */
 577        intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
 578        input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
 579                      (dram_addr & 0xfff);
 580
 581        edac_dbg(2, "  Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
 582                 intlv_shift, (unsigned long)dram_addr,
 583                 (unsigned long)input_addr);
 584
 585        return input_addr;
 586}
 587
 588/*
 589 * Translate the SysAddr represented by @sys_addr to an InputAddr.  It is
 590 * assumed that @sys_addr maps to the node given by mci.
 591 */
 592static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
 593{
 594        u64 input_addr;
 595
 596        input_addr =
 597            dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
 598
 599        edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
 600                 (unsigned long)sys_addr, (unsigned long)input_addr);
 601
 602        return input_addr;
 603}
 604
 605/* Map the Error address to a PAGE and PAGE OFFSET. */
 606static inline void error_address_to_page_and_offset(u64 error_address,
 607                                                    struct err_info *err)
 608{
 609        err->page = (u32) (error_address >> PAGE_SHIFT);
 610        err->offset = ((u32) error_address) & ~PAGE_MASK;
 611}
 612
 613/*
 614 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
 615 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
 616 * of a node that detected an ECC memory error.  mci represents the node that
 617 * the error address maps to (possibly different from the node that detected
 618 * the error).  Return the number of the csrow that sys_addr maps to, or -1 on
 619 * error.
 620 */
 621static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
 622{
 623        int csrow;
 624
 625        csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
 626
 627        if (csrow == -1)
 628                amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
 629                                  "address 0x%lx\n", (unsigned long)sys_addr);
 630        return csrow;
 631}
 632
 633static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
 634
 635/*
 636 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
 637 * are ECC capable.
 638 */
 639static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
 640{
 641        u8 bit;
 642        unsigned long edac_cap = EDAC_FLAG_NONE;
 643
 644        bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
 645                ? 19
 646                : 17;
 647
 648        if (pvt->dclr0 & BIT(bit))
 649                edac_cap = EDAC_FLAG_SECDED;
 650
 651        return edac_cap;
 652}
 653
 654static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
 655
 656static void amd64_dump_dramcfg_low(u32 dclr, int chan)
 657{
 658        edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
 659
 660        edac_dbg(1, "  DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
 661                 (dclr & BIT(16)) ?  "un" : "",
 662                 (dclr & BIT(19)) ? "yes" : "no");
 663
 664        edac_dbg(1, "  PAR/ERR parity: %s\n",
 665                 (dclr & BIT(8)) ?  "enabled" : "disabled");
 666
 667        if (boot_cpu_data.x86 == 0x10)
 668                edac_dbg(1, "  DCT 128bit mode width: %s\n",
 669                         (dclr & BIT(11)) ?  "128b" : "64b");
 670
 671        edac_dbg(1, "  x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
 672                 (dclr & BIT(12)) ?  "yes" : "no",
 673                 (dclr & BIT(13)) ?  "yes" : "no",
 674                 (dclr & BIT(14)) ?  "yes" : "no",
 675                 (dclr & BIT(15)) ?  "yes" : "no");
 676}
 677
 678/* Display and decode various NB registers for debug purposes. */
 679static void dump_misc_regs(struct amd64_pvt *pvt)
 680{
 681        edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
 682
 683        edac_dbg(1, "  NB two channel DRAM capable: %s\n",
 684                 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
 685
 686        edac_dbg(1, "  ECC capable: %s, ChipKill ECC capable: %s\n",
 687                 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
 688                 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
 689
 690        amd64_dump_dramcfg_low(pvt->dclr0, 0);
 691
 692        edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
 693
 694        edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
 695                 pvt->dhar, dhar_base(pvt),
 696                 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
 697                 : f10_dhar_offset(pvt));
 698
 699        edac_dbg(1, "  DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
 700
 701        amd64_debug_display_dimm_sizes(pvt, 0);
 702
 703        /* everything below this point is Fam10h and above */
 704        if (boot_cpu_data.x86 == 0xf)
 705                return;
 706
 707        amd64_debug_display_dimm_sizes(pvt, 1);
 708
 709        amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
 710
 711        /* Only if NOT ganged does dclr1 have valid info */
 712        if (!dct_ganging_enabled(pvt))
 713                amd64_dump_dramcfg_low(pvt->dclr1, 1);
 714}
 715
 716/*
 717 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
 718 */
 719static void prep_chip_selects(struct amd64_pvt *pvt)
 720{
 721        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
 722                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
 723                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
 724        } else {
 725                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
 726                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
 727        }
 728}
 729
 730/*
 731 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
 732 */
 733static void read_dct_base_mask(struct amd64_pvt *pvt)
 734{
 735        int cs;
 736
 737        prep_chip_selects(pvt);
 738
 739        for_each_chip_select(cs, 0, pvt) {
 740                int reg0   = DCSB0 + (cs * 4);
 741                int reg1   = DCSB1 + (cs * 4);
 742                u32 *base0 = &pvt->csels[0].csbases[cs];
 743                u32 *base1 = &pvt->csels[1].csbases[cs];
 744
 745                if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
 746                        edac_dbg(0, "  DCSB0[%d]=0x%08x reg: F2x%x\n",
 747                                 cs, *base0, reg0);
 748
 749                if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
 750                        continue;
 751
 752                if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
 753                        edac_dbg(0, "  DCSB1[%d]=0x%08x reg: F2x%x\n",
 754                                 cs, *base1, reg1);
 755        }
 756
 757        for_each_chip_select_mask(cs, 0, pvt) {
 758                int reg0   = DCSM0 + (cs * 4);
 759                int reg1   = DCSM1 + (cs * 4);
 760                u32 *mask0 = &pvt->csels[0].csmasks[cs];
 761                u32 *mask1 = &pvt->csels[1].csmasks[cs];
 762
 763                if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
 764                        edac_dbg(0, "    DCSM0[%d]=0x%08x reg: F2x%x\n",
 765                                 cs, *mask0, reg0);
 766
 767                if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
 768                        continue;
 769
 770                if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
 771                        edac_dbg(0, "    DCSM1[%d]=0x%08x reg: F2x%x\n",
 772                                 cs, *mask1, reg1);
 773        }
 774}
 775
 776static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
 777{
 778        enum mem_type type;
 779
 780        /* F15h supports only DDR3 */
 781        if (boot_cpu_data.x86 >= 0x15)
 782                type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
 783        else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
 784                if (pvt->dchr0 & DDR3_MODE)
 785                        type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
 786                else
 787                        type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
 788        } else {
 789                type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
 790        }
 791
 792        amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
 793
 794        return type;
 795}
 796
 797/* Get the number of DCT channels the memory controller is using. */
 798static int k8_early_channel_count(struct amd64_pvt *pvt)
 799{
 800        int flag;
 801
 802        if (pvt->ext_model >= K8_REV_F)
 803                /* RevF (NPT) and later */
 804                flag = pvt->dclr0 & WIDTH_128;
 805        else
 806                /* RevE and earlier */
 807                flag = pvt->dclr0 & REVE_WIDTH_128;
 808
 809        /* not used */
 810        pvt->dclr1 = 0;
 811
 812        return (flag) ? 2 : 1;
 813}
 814
 815/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
 816static u64 get_error_address(struct mce *m)
 817{
 818        struct cpuinfo_x86 *c = &boot_cpu_data;
 819        u64 addr;
 820        u8 start_bit = 1;
 821        u8 end_bit   = 47;
 822
 823        if (c->x86 == 0xf) {
 824                start_bit = 3;
 825                end_bit   = 39;
 826        }
 827
 828        addr = m->addr & GENMASK(start_bit, end_bit);
 829
 830        /*
 831         * Erratum 637 workaround
 832         */
 833        if (c->x86 == 0x15) {
 834                struct amd64_pvt *pvt;
 835                u64 cc6_base, tmp_addr;
 836                u32 tmp;
 837                u16 mce_nid;
 838                u8 intlv_en;
 839
 840                if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
 841                        return addr;
 842
 843                mce_nid = amd_get_nb_id(m->extcpu);
 844                pvt     = mcis[mce_nid]->pvt_info;
 845
 846                amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
 847                intlv_en = tmp >> 21 & 0x7;
 848
 849                /* add [47:27] + 3 trailing bits */
 850                cc6_base  = (tmp & GENMASK(0, 20)) << 3;
 851
 852                /* reverse and add DramIntlvEn */
 853                cc6_base |= intlv_en ^ 0x7;
 854
 855                /* pin at [47:24] */
 856                cc6_base <<= 24;
 857
 858                if (!intlv_en)
 859                        return cc6_base | (addr & GENMASK(0, 23));
 860
 861                amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
 862
 863                                                        /* faster log2 */
 864                tmp_addr  = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
 865
 866                /* OR DramIntlvSel into bits [14:12] */
 867                tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
 868
 869                /* add remaining [11:0] bits from original MC4_ADDR */
 870                tmp_addr |= addr & GENMASK(0, 11);
 871
 872                return cc6_base | tmp_addr;
 873        }
 874
 875        return addr;
 876}
 877
 878static struct pci_dev *pci_get_related_function(unsigned int vendor,
 879                                                unsigned int device,
 880                                                struct pci_dev *related)
 881{
 882        struct pci_dev *dev = NULL;
 883
 884        while ((dev = pci_get_device(vendor, device, dev))) {
 885                if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
 886                    (dev->bus->number == related->bus->number) &&
 887                    (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
 888                        break;
 889        }
 890
 891        return dev;
 892}
 893
 894static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
 895{
 896        struct amd_northbridge *nb;
 897        struct pci_dev *misc, *f1 = NULL;
 898        struct cpuinfo_x86 *c = &boot_cpu_data;
 899        int off = range << 3;
 900        u32 llim;
 901
 902        amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off,  &pvt->ranges[range].base.lo);
 903        amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
 904
 905        if (c->x86 == 0xf)
 906                return;
 907
 908        if (!dram_rw(pvt, range))
 909                return;
 910
 911        amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off,  &pvt->ranges[range].base.hi);
 912        amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
 913
 914        /* F15h: factor in CC6 save area by reading dst node's limit reg */
 915        if (c->x86 != 0x15)
 916                return;
 917
 918        nb = node_to_amd_nb(dram_dst_node(pvt, range));
 919        if (WARN_ON(!nb))
 920                return;
 921
 922        misc = nb->misc;
 923        f1 = pci_get_related_function(misc->vendor, PCI_DEVICE_ID_AMD_15H_NB_F1, misc);
 924        if (WARN_ON(!f1))
 925                return;
 926
 927        amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
 928
 929        pvt->ranges[range].lim.lo &= GENMASK(0, 15);
 930
 931                                    /* {[39:27],111b} */
 932        pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
 933
 934        pvt->ranges[range].lim.hi &= GENMASK(0, 7);
 935
 936                                    /* [47:40] */
 937        pvt->ranges[range].lim.hi |= llim >> 13;
 938
 939        pci_dev_put(f1);
 940}
 941
 942static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
 943                                    struct err_info *err)
 944{
 945        struct amd64_pvt *pvt = mci->pvt_info;
 946
 947        error_address_to_page_and_offset(sys_addr, err);
 948
 949        /*
 950         * Find out which node the error address belongs to. This may be
 951         * different from the node that detected the error.
 952         */
 953        err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
 954        if (!err->src_mci) {
 955                amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
 956                             (unsigned long)sys_addr);
 957                err->err_code = ERR_NODE;
 958                return;
 959        }
 960
 961        /* Now map the sys_addr to a CSROW */
 962        err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
 963        if (err->csrow < 0) {
 964                err->err_code = ERR_CSROW;
 965                return;
 966        }
 967
 968        /* CHIPKILL enabled */
 969        if (pvt->nbcfg & NBCFG_CHIPKILL) {
 970                err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
 971                if (err->channel < 0) {
 972                        /*
 973                         * Syndrome didn't map, so we don't know which of the
 974                         * 2 DIMMs is in error. So we need to ID 'both' of them
 975                         * as suspect.
 976                         */
 977                        amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
 978                                      "possible error reporting race\n",
 979                                      err->syndrome);
 980                        err->err_code = ERR_CHANNEL;
 981                        return;
 982                }
 983        } else {
 984                /*
 985                 * non-chipkill ecc mode
 986                 *
 987                 * The k8 documentation is unclear about how to determine the
 988                 * channel number when using non-chipkill memory.  This method
 989                 * was obtained from email communication with someone at AMD.
 990                 * (Wish the email was placed in this comment - norsk)
 991                 */
 992                err->channel = ((sys_addr & BIT(3)) != 0);
 993        }
 994}
 995
 996static int ddr2_cs_size(unsigned i, bool dct_width)
 997{
 998        unsigned shift = 0;
 999
1000        if (i <= 2)
1001                shift = i;
1002        else if (!(i & 0x1))
1003                shift = i >> 1;
1004        else
1005                shift = (i + 1) >> 1;
1006
1007        return 128 << (shift + !!dct_width);
1008}
1009
1010static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1011                                  unsigned cs_mode)
1012{
1013        u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1014
1015        if (pvt->ext_model >= K8_REV_F) {
1016                WARN_ON(cs_mode > 11);
1017                return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1018        }
1019        else if (pvt->ext_model >= K8_REV_D) {
1020                unsigned diff;
1021                WARN_ON(cs_mode > 10);
1022
1023                /*
1024                 * the below calculation, besides trying to win an obfuscated C
1025                 * contest, maps cs_mode values to DIMM chip select sizes. The
1026                 * mappings are:
1027                 *
1028                 * cs_mode      CS size (mb)
1029                 * =======      ============
1030                 * 0            32
1031                 * 1            64
1032                 * 2            128
1033                 * 3            128
1034                 * 4            256
1035                 * 5            512
1036                 * 6            256
1037                 * 7            512
1038                 * 8            1024
1039                 * 9            1024
1040                 * 10           2048
1041                 *
1042                 * Basically, it calculates a value with which to shift the
1043                 * smallest CS size of 32MB.
1044                 *
1045                 * ddr[23]_cs_size have a similar purpose.
1046                 */
1047                diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1048
1049                return 32 << (cs_mode - diff);
1050        }
1051        else {
1052                WARN_ON(cs_mode > 6);
1053                return 32 << cs_mode;
1054        }
1055}
1056
1057/*
1058 * Get the number of DCT channels in use.
1059 *
1060 * Return:
1061 *      number of Memory Channels in operation
1062 * Pass back:
1063 *      contents of the DCL0_LOW register
1064 */
1065static int f1x_early_channel_count(struct amd64_pvt *pvt)
1066{
1067        int i, j, channels = 0;
1068
1069        /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1070        if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1071                return 2;
1072
1073        /*
1074         * Need to check if in unganged mode: In such, there are 2 channels,
1075         * but they are not in 128 bit mode and thus the above 'dclr0' status
1076         * bit will be OFF.
1077         *
1078         * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1079         * their CSEnable bit on. If so, then SINGLE DIMM case.
1080         */
1081        edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1082
1083        /*
1084         * Check DRAM Bank Address Mapping values for each DIMM to see if there
1085         * is more than just one DIMM present in unganged mode. Need to check
1086         * both controllers since DIMMs can be placed in either one.
1087         */
1088        for (i = 0; i < 2; i++) {
1089                u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1090
1091                for (j = 0; j < 4; j++) {
1092                        if (DBAM_DIMM(j, dbam) > 0) {
1093                                channels++;
1094                                break;
1095                        }
1096                }
1097        }
1098
1099        if (channels > 2)
1100                channels = 2;
1101
1102        amd64_info("MCT channel count: %d\n", channels);
1103
1104        return channels;
1105}
1106
1107static int ddr3_cs_size(unsigned i, bool dct_width)
1108{
1109        unsigned shift = 0;
1110        int cs_size = 0;
1111
1112        if (i == 0 || i == 3 || i == 4)
1113                cs_size = -1;
1114        else if (i <= 2)
1115                shift = i;
1116        else if (i == 12)
1117                shift = 7;
1118        else if (!(i & 0x1))
1119                shift = i >> 1;
1120        else
1121                shift = (i + 1) >> 1;
1122
1123        if (cs_size != -1)
1124                cs_size = (128 * (1 << !!dct_width)) << shift;
1125
1126        return cs_size;
1127}
1128
1129static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1130                                   unsigned cs_mode)
1131{
1132        u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1133
1134        WARN_ON(cs_mode > 11);
1135
1136        if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1137                return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1138        else
1139                return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1140}
1141
1142/*
1143 * F15h supports only 64bit DCT interfaces
1144 */
1145static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1146                                   unsigned cs_mode)
1147{
1148        WARN_ON(cs_mode > 12);
1149
1150        return ddr3_cs_size(cs_mode, false);
1151}
1152
1153static void read_dram_ctl_register(struct amd64_pvt *pvt)
1154{
1155
1156        if (boot_cpu_data.x86 == 0xf)
1157                return;
1158
1159        if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1160                edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1161                         pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1162
1163                edac_dbg(0, "  DCTs operate in %s mode\n",
1164                         (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1165
1166                if (!dct_ganging_enabled(pvt))
1167                        edac_dbg(0, "  Address range split per DCT: %s\n",
1168                                 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1169
1170                edac_dbg(0, "  data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1171                         (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1172                         (dct_memory_cleared(pvt) ? "yes" : "no"));
1173
1174                edac_dbg(0, "  channel interleave: %s, "
1175                         "interleave bits selector: 0x%x\n",
1176                         (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1177                         dct_sel_interleave_addr(pvt));
1178        }
1179
1180        amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1181}
1182
1183/*
1184 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1185 * Interleaving Modes.
1186 */
1187static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1188                                bool hi_range_sel, u8 intlv_en)
1189{
1190        u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1191
1192        if (dct_ganging_enabled(pvt))
1193                return 0;
1194
1195        if (hi_range_sel)
1196                return dct_sel_high;
1197
1198        /*
1199         * see F2x110[DctSelIntLvAddr] - channel interleave mode
1200         */
1201        if (dct_interleave_enabled(pvt)) {
1202                u8 intlv_addr = dct_sel_interleave_addr(pvt);
1203
1204                /* return DCT select function: 0=DCT0, 1=DCT1 */
1205                if (!intlv_addr)
1206                        return sys_addr >> 6 & 1;
1207
1208                if (intlv_addr & 0x2) {
1209                        u8 shift = intlv_addr & 0x1 ? 9 : 6;
1210                        u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1211
1212                        return ((sys_addr >> shift) & 1) ^ temp;
1213                }
1214
1215                return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1216        }
1217
1218        if (dct_high_range_enabled(pvt))
1219                return ~dct_sel_high & 1;
1220
1221        return 0;
1222}
1223
1224/* Convert the sys_addr to the normalized DCT address */
1225static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1226                                 u64 sys_addr, bool hi_rng,
1227                                 u32 dct_sel_base_addr)
1228{
1229        u64 chan_off;
1230        u64 dram_base           = get_dram_base(pvt, range);
1231        u64 hole_off            = f10_dhar_offset(pvt);
1232        u64 dct_sel_base_off    = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1233
1234        if (hi_rng) {
1235                /*
1236                 * if
1237                 * base address of high range is below 4Gb
1238                 * (bits [47:27] at [31:11])
1239                 * DRAM address space on this DCT is hoisted above 4Gb  &&
1240                 * sys_addr > 4Gb
1241                 *
1242                 *      remove hole offset from sys_addr
1243                 * else
1244                 *      remove high range offset from sys_addr
1245                 */
1246                if ((!(dct_sel_base_addr >> 16) ||
1247                     dct_sel_base_addr < dhar_base(pvt)) &&
1248                    dhar_valid(pvt) &&
1249                    (sys_addr >= BIT_64(32)))
1250                        chan_off = hole_off;
1251                else
1252                        chan_off = dct_sel_base_off;
1253        } else {
1254                /*
1255                 * if
1256                 * we have a valid hole         &&
1257                 * sys_addr > 4Gb
1258                 *
1259                 *      remove hole
1260                 * else
1261                 *      remove dram base to normalize to DCT address
1262                 */
1263                if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1264                        chan_off = hole_off;
1265                else
1266                        chan_off = dram_base;
1267        }
1268
1269        return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1270}
1271
1272/*
1273 * checks if the csrow passed in is marked as SPARED, if so returns the new
1274 * spare row
1275 */
1276static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1277{
1278        int tmp_cs;
1279
1280        if (online_spare_swap_done(pvt, dct) &&
1281            csrow == online_spare_bad_dramcs(pvt, dct)) {
1282
1283                for_each_chip_select(tmp_cs, dct, pvt) {
1284                        if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1285                                csrow = tmp_cs;
1286                                break;
1287                        }
1288                }
1289        }
1290        return csrow;
1291}
1292
1293/*
1294 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1295 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1296 *
1297 * Return:
1298 *      -EINVAL:  NOT FOUND
1299 *      0..csrow = Chip-Select Row
1300 */
1301static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1302{
1303        struct mem_ctl_info *mci;
1304        struct amd64_pvt *pvt;
1305        u64 cs_base, cs_mask;
1306        int cs_found = -EINVAL;
1307        int csrow;
1308
1309        mci = mcis[nid];
1310        if (!mci)
1311                return cs_found;
1312
1313        pvt = mci->pvt_info;
1314
1315        edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1316
1317        for_each_chip_select(csrow, dct, pvt) {
1318                if (!csrow_enabled(csrow, dct, pvt))
1319                        continue;
1320
1321                get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1322
1323                edac_dbg(1, "    CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1324                         csrow, cs_base, cs_mask);
1325
1326                cs_mask = ~cs_mask;
1327
1328                edac_dbg(1, "    (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1329                         (in_addr & cs_mask), (cs_base & cs_mask));
1330
1331                if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1332                        cs_found = f10_process_possible_spare(pvt, dct, csrow);
1333
1334                        edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1335                        break;
1336                }
1337        }
1338        return cs_found;
1339}
1340
1341/*
1342 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1343 * swapped with a region located at the bottom of memory so that the GPU can use
1344 * the interleaved region and thus two channels.
1345 */
1346static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1347{
1348        u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1349
1350        if (boot_cpu_data.x86 == 0x10) {
1351                /* only revC3 and revE have that feature */
1352                if (boot_cpu_data.x86_model < 4 ||
1353                    (boot_cpu_data.x86_model < 0xa &&
1354                     boot_cpu_data.x86_mask < 3))
1355                        return sys_addr;
1356        }
1357
1358        amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1359
1360        if (!(swap_reg & 0x1))
1361                return sys_addr;
1362
1363        swap_base       = (swap_reg >> 3) & 0x7f;
1364        swap_limit      = (swap_reg >> 11) & 0x7f;
1365        rgn_size        = (swap_reg >> 20) & 0x7f;
1366        tmp_addr        = sys_addr >> 27;
1367
1368        if (!(sys_addr >> 34) &&
1369            (((tmp_addr >= swap_base) &&
1370             (tmp_addr <= swap_limit)) ||
1371             (tmp_addr < rgn_size)))
1372                return sys_addr ^ (u64)swap_base << 27;
1373
1374        return sys_addr;
1375}
1376
1377/* For a given @dram_range, check if @sys_addr falls within it. */
1378static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1379                                  u64 sys_addr, int *chan_sel)
1380{
1381        int cs_found = -EINVAL;
1382        u64 chan_addr;
1383        u32 dct_sel_base;
1384        u8 channel;
1385        bool high_range = false;
1386
1387        u8 node_id    = dram_dst_node(pvt, range);
1388        u8 intlv_en   = dram_intlv_en(pvt, range);
1389        u32 intlv_sel = dram_intlv_sel(pvt, range);
1390
1391        edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1392                 range, sys_addr, get_dram_limit(pvt, range));
1393
1394        if (dhar_valid(pvt) &&
1395            dhar_base(pvt) <= sys_addr &&
1396            sys_addr < BIT_64(32)) {
1397                amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1398                            sys_addr);
1399                return -EINVAL;
1400        }
1401
1402        if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1403                return -EINVAL;
1404
1405        sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1406
1407        dct_sel_base = dct_sel_baseaddr(pvt);
1408
1409        /*
1410         * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1411         * select between DCT0 and DCT1.
1412         */
1413        if (dct_high_range_enabled(pvt) &&
1414           !dct_ganging_enabled(pvt) &&
1415           ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1416                high_range = true;
1417
1418        channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1419
1420        chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1421                                          high_range, dct_sel_base);
1422
1423        /* Remove node interleaving, see F1x120 */
1424        if (intlv_en)
1425                chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1426                            (chan_addr & 0xfff);
1427
1428        /* remove channel interleave */
1429        if (dct_interleave_enabled(pvt) &&
1430           !dct_high_range_enabled(pvt) &&
1431           !dct_ganging_enabled(pvt)) {
1432
1433                if (dct_sel_interleave_addr(pvt) != 1) {
1434                        if (dct_sel_interleave_addr(pvt) == 0x3)
1435                                /* hash 9 */
1436                                chan_addr = ((chan_addr >> 10) << 9) |
1437                                             (chan_addr & 0x1ff);
1438                        else
1439                                /* A[6] or hash 6 */
1440                                chan_addr = ((chan_addr >> 7) << 6) |
1441                                             (chan_addr & 0x3f);
1442                } else
1443                        /* A[12] */
1444                        chan_addr = ((chan_addr >> 13) << 12) |
1445                                     (chan_addr & 0xfff);
1446        }
1447
1448        edac_dbg(1, "   Normalized DCT addr: 0x%llx\n", chan_addr);
1449
1450        cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1451
1452        if (cs_found >= 0)
1453                *chan_sel = channel;
1454
1455        return cs_found;
1456}
1457
1458static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1459                                       int *chan_sel)
1460{
1461        int cs_found = -EINVAL;
1462        unsigned range;
1463
1464        for (range = 0; range < DRAM_RANGES; range++) {
1465
1466                if (!dram_rw(pvt, range))
1467                        continue;
1468
1469                if ((get_dram_base(pvt, range)  <= sys_addr) &&
1470                    (get_dram_limit(pvt, range) >= sys_addr)) {
1471
1472                        cs_found = f1x_match_to_this_node(pvt, range,
1473                                                          sys_addr, chan_sel);
1474                        if (cs_found >= 0)
1475                                break;
1476                }
1477        }
1478        return cs_found;
1479}
1480
1481/*
1482 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1483 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1484 *
1485 * The @sys_addr is usually an error address received from the hardware
1486 * (MCX_ADDR).
1487 */
1488static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1489                                     struct err_info *err)
1490{
1491        struct amd64_pvt *pvt = mci->pvt_info;
1492
1493        error_address_to_page_and_offset(sys_addr, err);
1494
1495        err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1496        if (err->csrow < 0) {
1497                err->err_code = ERR_CSROW;
1498                return;
1499        }
1500
1501        /*
1502         * We need the syndromes for channel detection only when we're
1503         * ganged. Otherwise @chan should already contain the channel at
1504         * this point.
1505         */
1506        if (dct_ganging_enabled(pvt))
1507                err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1508}
1509
1510/*
1511 * debug routine to display the memory sizes of all logical DIMMs and its
1512 * CSROWs
1513 */
1514static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1515{
1516        int dimm, size0, size1;
1517        u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1518        u32 dbam  = ctrl ? pvt->dbam1 : pvt->dbam0;
1519
1520        if (boot_cpu_data.x86 == 0xf) {
1521                /* K8 families < revF not supported yet */
1522               if (pvt->ext_model < K8_REV_F)
1523                        return;
1524               else
1525                       WARN_ON(ctrl != 0);
1526        }
1527
1528        dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1529        dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1530                                                   : pvt->csels[0].csbases;
1531
1532        edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1533                 ctrl, dbam);
1534
1535        edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1536
1537        /* Dump memory sizes for DIMM and its CSROWs */
1538        for (dimm = 0; dimm < 4; dimm++) {
1539
1540                size0 = 0;
1541                if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1542                        size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1543                                                     DBAM_DIMM(dimm, dbam));
1544
1545                size1 = 0;
1546                if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1547                        size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1548                                                     DBAM_DIMM(dimm, dbam));
1549
1550                amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1551                                dimm * 2,     size0,
1552                                dimm * 2 + 1, size1);
1553        }
1554}
1555
1556static struct amd64_family_type amd64_family_types[] = {
1557        [K8_CPUS] = {
1558                .ctl_name = "K8",
1559                .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1560                .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1561                .ops = {
1562                        .early_channel_count    = k8_early_channel_count,
1563                        .map_sysaddr_to_csrow   = k8_map_sysaddr_to_csrow,
1564                        .dbam_to_cs             = k8_dbam_to_chip_select,
1565                        .read_dct_pci_cfg       = k8_read_dct_pci_cfg,
1566                }
1567        },
1568        [F10_CPUS] = {
1569                .ctl_name = "F10h",
1570                .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1571                .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1572                .ops = {
1573                        .early_channel_count    = f1x_early_channel_count,
1574                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
1575                        .dbam_to_cs             = f10_dbam_to_chip_select,
1576                        .read_dct_pci_cfg       = f10_read_dct_pci_cfg,
1577                }
1578        },
1579        [F15_CPUS] = {
1580                .ctl_name = "F15h",
1581                .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1582                .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1583                .ops = {
1584                        .early_channel_count    = f1x_early_channel_count,
1585                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
1586                        .dbam_to_cs             = f15_dbam_to_chip_select,
1587                        .read_dct_pci_cfg       = f15_read_dct_pci_cfg,
1588                }
1589        },
1590};
1591
1592/*
1593 * These are tables of eigenvectors (one per line) which can be used for the
1594 * construction of the syndrome tables. The modified syndrome search algorithm
1595 * uses those to find the symbol in error and thus the DIMM.
1596 *
1597 * Algorithm courtesy of Ross LaFetra from AMD.
1598 */
1599static const u16 x4_vectors[] = {
1600        0x2f57, 0x1afe, 0x66cc, 0xdd88,
1601        0x11eb, 0x3396, 0x7f4c, 0xeac8,
1602        0x0001, 0x0002, 0x0004, 0x0008,
1603        0x1013, 0x3032, 0x4044, 0x8088,
1604        0x106b, 0x30d6, 0x70fc, 0xe0a8,
1605        0x4857, 0xc4fe, 0x13cc, 0x3288,
1606        0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1607        0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1608        0x15c1, 0x2a42, 0x89ac, 0x4758,
1609        0x2b03, 0x1602, 0x4f0c, 0xca08,
1610        0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1611        0x8ba7, 0x465e, 0x244c, 0x1cc8,
1612        0x2b87, 0x164e, 0x642c, 0xdc18,
1613        0x40b9, 0x80de, 0x1094, 0x20e8,
1614        0x27db, 0x1eb6, 0x9dac, 0x7b58,
1615        0x11c1, 0x2242, 0x84ac, 0x4c58,
1616        0x1be5, 0x2d7a, 0x5e34, 0xa718,
1617        0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1618        0x4c97, 0xc87e, 0x11fc, 0x33a8,
1619        0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1620        0x16b3, 0x3d62, 0x4f34, 0x8518,
1621        0x1e2f, 0x391a, 0x5cac, 0xf858,
1622        0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1623        0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1624        0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1625        0x4397, 0xc27e, 0x17fc, 0x3ea8,
1626        0x1617, 0x3d3e, 0x6464, 0xb8b8,
1627        0x23ff, 0x12aa, 0xab6c, 0x56d8,
1628        0x2dfb, 0x1ba6, 0x913c, 0x7328,
1629        0x185d, 0x2ca6, 0x7914, 0x9e28,
1630        0x171b, 0x3e36, 0x7d7c, 0xebe8,
1631        0x4199, 0x82ee, 0x19f4, 0x2e58,
1632        0x4807, 0xc40e, 0x130c, 0x3208,
1633        0x1905, 0x2e0a, 0x5804, 0xac08,
1634        0x213f, 0x132a, 0xadfc, 0x5ba8,
1635        0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1636};
1637
1638static const u16 x8_vectors[] = {
1639        0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1640        0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1641        0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1642        0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1643        0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1644        0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1645        0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1646        0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1647        0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1648        0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1649        0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1650        0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1651        0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1652        0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1653        0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1654        0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1655        0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1656        0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1657        0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1658};
1659
1660static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
1661                           unsigned v_dim)
1662{
1663        unsigned int i, err_sym;
1664
1665        for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1666                u16 s = syndrome;
1667                unsigned v_idx =  err_sym * v_dim;
1668                unsigned v_end = (err_sym + 1) * v_dim;
1669
1670                /* walk over all 16 bits of the syndrome */
1671                for (i = 1; i < (1U << 16); i <<= 1) {
1672
1673                        /* if bit is set in that eigenvector... */
1674                        if (v_idx < v_end && vectors[v_idx] & i) {
1675                                u16 ev_comp = vectors[v_idx++];
1676
1677                                /* ... and bit set in the modified syndrome, */
1678                                if (s & i) {
1679                                        /* remove it. */
1680                                        s ^= ev_comp;
1681
1682                                        if (!s)
1683                                                return err_sym;
1684                                }
1685
1686                        } else if (s & i)
1687                                /* can't get to zero, move to next symbol */
1688                                break;
1689                }
1690        }
1691
1692        edac_dbg(0, "syndrome(%x) not found\n", syndrome);
1693        return -1;
1694}
1695
1696static int map_err_sym_to_channel(int err_sym, int sym_size)
1697{
1698        if (sym_size == 4)
1699                switch (err_sym) {
1700                case 0x20:
1701                case 0x21:
1702                        return 0;
1703                        break;
1704                case 0x22:
1705                case 0x23:
1706                        return 1;
1707                        break;
1708                default:
1709                        return err_sym >> 4;
1710                        break;
1711                }
1712        /* x8 symbols */
1713        else
1714                switch (err_sym) {
1715                /* imaginary bits not in a DIMM */
1716                case 0x10:
1717                        WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1718                                          err_sym);
1719                        return -1;
1720                        break;
1721
1722                case 0x11:
1723                        return 0;
1724                        break;
1725                case 0x12:
1726                        return 1;
1727                        break;
1728                default:
1729                        return err_sym >> 3;
1730                        break;
1731                }
1732        return -1;
1733}
1734
1735static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1736{
1737        struct amd64_pvt *pvt = mci->pvt_info;
1738        int err_sym = -1;
1739
1740        if (pvt->ecc_sym_sz == 8)
1741                err_sym = decode_syndrome(syndrome, x8_vectors,
1742                                          ARRAY_SIZE(x8_vectors),
1743                                          pvt->ecc_sym_sz);
1744        else if (pvt->ecc_sym_sz == 4)
1745                err_sym = decode_syndrome(syndrome, x4_vectors,
1746                                          ARRAY_SIZE(x4_vectors),
1747                                          pvt->ecc_sym_sz);
1748        else {
1749                amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1750                return err_sym;
1751        }
1752
1753        return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1754}
1755
1756static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
1757                            u8 ecc_type)
1758{
1759        enum hw_event_mc_err_type err_type;
1760        const char *string;
1761
1762        if (ecc_type == 2)
1763                err_type = HW_EVENT_ERR_CORRECTED;
1764        else if (ecc_type == 1)
1765                err_type = HW_EVENT_ERR_UNCORRECTED;
1766        else {
1767                WARN(1, "Something is rotten in the state of Denmark.\n");
1768                return;
1769        }
1770
1771        switch (err->err_code) {
1772        case DECODE_OK:
1773                string = "";
1774                break;
1775        case ERR_NODE:
1776                string = "Failed to map error addr to a node";
1777                break;
1778        case ERR_CSROW:
1779                string = "Failed to map error addr to a csrow";
1780                break;
1781        case ERR_CHANNEL:
1782                string = "unknown syndrome - possible error reporting race";
1783                break;
1784        default:
1785                string = "WTF error";
1786                break;
1787        }
1788
1789        edac_mc_handle_error(err_type, mci, 1,
1790                             err->page, err->offset, err->syndrome,
1791                             err->csrow, err->channel, -1,
1792                             string, "");
1793}
1794
1795static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1796                                            struct mce *m)
1797{
1798        struct amd64_pvt *pvt = mci->pvt_info;
1799        u8 ecc_type = (m->status >> 45) & 0x3;
1800        u8 xec = XEC(m->status, 0x1f);
1801        u16 ec = EC(m->status);
1802        u64 sys_addr;
1803        struct err_info err;
1804
1805        /* Bail out early if this was an 'observed' error */
1806        if (PP(ec) == NBSL_PP_OBS)
1807                return;
1808
1809        /* Do only ECC errors */
1810        if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1811                return;
1812
1813        memset(&err, 0, sizeof(err));
1814
1815        sys_addr = get_error_address(m);
1816
1817        if (ecc_type == 2)
1818                err.syndrome = extract_syndrome(m->status);
1819
1820        pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
1821
1822        __log_bus_error(mci, &err, ecc_type);
1823}
1824
1825void amd64_decode_bus_error(int node_id, struct mce *m)
1826{
1827        __amd64_decode_bus_error(mcis[node_id], m);
1828}
1829
1830/*
1831 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1832 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1833 */
1834static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1835{
1836        /* Reserve the ADDRESS MAP Device */
1837        pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1838        if (!pvt->F1) {
1839                amd64_err("error address map device not found: "
1840                          "vendor %x device 0x%x (broken BIOS?)\n",
1841                          PCI_VENDOR_ID_AMD, f1_id);
1842                return -ENODEV;
1843        }
1844
1845        /* Reserve the MISC Device */
1846        pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1847        if (!pvt->F3) {
1848                pci_dev_put(pvt->F1);
1849                pvt->F1 = NULL;
1850
1851                amd64_err("error F3 device not found: "
1852                          "vendor %x device 0x%x (broken BIOS?)\n",
1853                          PCI_VENDOR_ID_AMD, f3_id);
1854
1855                return -ENODEV;
1856        }
1857        edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
1858        edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
1859        edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
1860
1861        return 0;
1862}
1863
1864static void free_mc_sibling_devs(struct amd64_pvt *pvt)
1865{
1866        pci_dev_put(pvt->F1);
1867        pci_dev_put(pvt->F3);
1868}
1869
1870/*
1871 * Retrieve the hardware registers of the memory controller (this includes the
1872 * 'Address Map' and 'Misc' device regs)
1873 */
1874static void read_mc_regs(struct amd64_pvt *pvt)
1875{
1876        struct cpuinfo_x86 *c = &boot_cpu_data;
1877        u64 msr_val;
1878        u32 tmp;
1879        unsigned range;
1880
1881        /*
1882         * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1883         * those are Read-As-Zero
1884         */
1885        rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
1886        edac_dbg(0, "  TOP_MEM:  0x%016llx\n", pvt->top_mem);
1887
1888        /* check first whether TOP_MEM2 is enabled */
1889        rdmsrl(MSR_K8_SYSCFG, msr_val);
1890        if (msr_val & (1U << 21)) {
1891                rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
1892                edac_dbg(0, "  TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
1893        } else
1894                edac_dbg(0, "  TOP_MEM2 disabled\n");
1895
1896        amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
1897
1898        read_dram_ctl_register(pvt);
1899
1900        for (range = 0; range < DRAM_RANGES; range++) {
1901                u8 rw;
1902
1903                /* read settings for this DRAM range */
1904                read_dram_base_limit_regs(pvt, range);
1905
1906                rw = dram_rw(pvt, range);
1907                if (!rw)
1908                        continue;
1909
1910                edac_dbg(1, "  DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1911                         range,
1912                         get_dram_base(pvt, range),
1913                         get_dram_limit(pvt, range));
1914
1915                edac_dbg(1, "   IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1916                         dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1917                         (rw & 0x1) ? "R" : "-",
1918                         (rw & 0x2) ? "W" : "-",
1919                         dram_intlv_sel(pvt, range),
1920                         dram_dst_node(pvt, range));
1921        }
1922
1923        read_dct_base_mask(pvt);
1924
1925        amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
1926        amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
1927
1928        amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
1929
1930        amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
1931        amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
1932
1933        if (!dct_ganging_enabled(pvt)) {
1934                amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
1935                amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
1936        }
1937
1938        pvt->ecc_sym_sz = 4;
1939
1940        if (c->x86 >= 0x10) {
1941                amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
1942                amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
1943
1944                /* F10h, revD and later can do x8 ECC too */
1945                if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
1946                        pvt->ecc_sym_sz = 8;
1947        }
1948        dump_misc_regs(pvt);
1949}
1950
1951/*
1952 * NOTE: CPU Revision Dependent code
1953 *
1954 * Input:
1955 *      @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
1956 *      k8 private pointer to -->
1957 *                      DRAM Bank Address mapping register
1958 *                      node_id
1959 *                      DCL register where dual_channel_active is
1960 *
1961 * The DBAM register consists of 4 sets of 4 bits each definitions:
1962 *
1963 * Bits:        CSROWs
1964 * 0-3          CSROWs 0 and 1
1965 * 4-7          CSROWs 2 and 3
1966 * 8-11         CSROWs 4 and 5
1967 * 12-15        CSROWs 6 and 7
1968 *
1969 * Values range from: 0 to 15
1970 * The meaning of the values depends on CPU revision and dual-channel state,
1971 * see relevant BKDG more info.
1972 *
1973 * The memory controller provides for total of only 8 CSROWs in its current
1974 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
1975 * single channel or two (2) DIMMs in dual channel mode.
1976 *
1977 * The following code logic collapses the various tables for CSROW based on CPU
1978 * revision.
1979 *
1980 * Returns:
1981 *      The number of PAGE_SIZE pages on the specified CSROW number it
1982 *      encompasses
1983 *
1984 */
1985static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
1986{
1987        u32 cs_mode, nr_pages;
1988        u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
1989
1990
1991        /*
1992         * The math on this doesn't look right on the surface because x/2*4 can
1993         * be simplified to x*2 but this expression makes use of the fact that
1994         * it is integral math where 1/2=0. This intermediate value becomes the
1995         * number of bits to shift the DBAM register to extract the proper CSROW
1996         * field.
1997         */
1998        cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
1999
2000        nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2001
2002        edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2003                    csrow_nr, dct,  cs_mode);
2004        edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2005
2006        return nr_pages;
2007}
2008
2009/*
2010 * Initialize the array of csrow attribute instances, based on the values
2011 * from pci config hardware registers.
2012 */
2013static int init_csrows(struct mem_ctl_info *mci)
2014{
2015        struct amd64_pvt *pvt = mci->pvt_info;
2016        struct csrow_info *csrow;
2017        struct dimm_info *dimm;
2018        enum edac_type edac_mode;
2019        enum mem_type mtype;
2020        int i, j, empty = 1;
2021        int nr_pages = 0;
2022        u32 val;
2023
2024        amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2025
2026        pvt->nbcfg = val;
2027
2028        edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2029                 pvt->mc_node_id, val,
2030                 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2031
2032        /*
2033         * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2034         */
2035        for_each_chip_select(i, 0, pvt) {
2036                bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2037                bool row_dct1 = false;
2038
2039                if (boot_cpu_data.x86 != 0xf)
2040                        row_dct1 = !!csrow_enabled(i, 1, pvt);
2041
2042                if (!row_dct0 && !row_dct1)
2043                        continue;
2044
2045                csrow = mci->csrows[i];
2046                empty = 0;
2047
2048                edac_dbg(1, "MC node: %d, csrow: %d\n",
2049                            pvt->mc_node_id, i);
2050
2051                if (row_dct0) {
2052                        nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2053                        csrow->channels[0]->dimm->nr_pages = nr_pages;
2054                }
2055
2056                /* K8 has only one DCT */
2057                if (boot_cpu_data.x86 != 0xf && row_dct1) {
2058                        int row_dct1_pages = amd64_csrow_nr_pages(pvt, 1, i);
2059
2060                        csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2061                        nr_pages += row_dct1_pages;
2062                }
2063
2064                mtype = amd64_determine_memory_type(pvt, i);
2065
2066                edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2067
2068                /*
2069                 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2070                 */
2071                if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2072                        edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2073                                    EDAC_S4ECD4ED : EDAC_SECDED;
2074                else
2075                        edac_mode = EDAC_NONE;
2076
2077                for (j = 0; j < pvt->channel_count; j++) {
2078                        dimm = csrow->channels[j]->dimm;
2079                        dimm->mtype = mtype;
2080                        dimm->edac_mode = edac_mode;
2081                }
2082        }
2083
2084        return empty;
2085}
2086
2087/* get all cores on this DCT */
2088static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2089{
2090        int cpu;
2091
2092        for_each_online_cpu(cpu)
2093                if (amd_get_nb_id(cpu) == nid)
2094                        cpumask_set_cpu(cpu, mask);
2095}
2096
2097/* check MCG_CTL on all the cpus on this node */
2098static bool amd64_nb_mce_bank_enabled_on_node(u16 nid)
2099{
2100        cpumask_var_t mask;
2101        int cpu, nbe;
2102        bool ret = false;
2103
2104        if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2105                amd64_warn("%s: Error allocating mask\n", __func__);
2106                return false;
2107        }
2108
2109        get_cpus_on_this_dct_cpumask(mask, nid);
2110
2111        rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2112
2113        for_each_cpu(cpu, mask) {
2114                struct msr *reg = per_cpu_ptr(msrs, cpu);
2115                nbe = reg->l & MSR_MCGCTL_NBE;
2116
2117                edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2118                         cpu, reg->q,
2119                         (nbe ? "enabled" : "disabled"));
2120
2121                if (!nbe)
2122                        goto out;
2123        }
2124        ret = true;
2125
2126out:
2127        free_cpumask_var(mask);
2128        return ret;
2129}
2130
2131static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2132{
2133        cpumask_var_t cmask;
2134        int cpu;
2135
2136        if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2137                amd64_warn("%s: error allocating mask\n", __func__);
2138                return false;
2139        }
2140
2141        get_cpus_on_this_dct_cpumask(cmask, nid);
2142
2143        rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2144
2145        for_each_cpu(cpu, cmask) {
2146
2147                struct msr *reg = per_cpu_ptr(msrs, cpu);
2148
2149                if (on) {
2150                        if (reg->l & MSR_MCGCTL_NBE)
2151                                s->flags.nb_mce_enable = 1;
2152
2153                        reg->l |= MSR_MCGCTL_NBE;
2154                } else {
2155                        /*
2156                         * Turn off NB MCE reporting only when it was off before
2157                         */
2158                        if (!s->flags.nb_mce_enable)
2159                                reg->l &= ~MSR_MCGCTL_NBE;
2160                }
2161        }
2162        wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2163
2164        free_cpumask_var(cmask);
2165
2166        return 0;
2167}
2168
2169static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2170                                       struct pci_dev *F3)
2171{
2172        bool ret = true;
2173        u32 value, mask = 0x3;          /* UECC/CECC enable */
2174
2175        if (toggle_ecc_err_reporting(s, nid, ON)) {
2176                amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2177                return false;
2178        }
2179
2180        amd64_read_pci_cfg(F3, NBCTL, &value);
2181
2182        s->old_nbctl   = value & mask;
2183        s->nbctl_valid = true;
2184
2185        value |= mask;
2186        amd64_write_pci_cfg(F3, NBCTL, value);
2187
2188        amd64_read_pci_cfg(F3, NBCFG, &value);
2189
2190        edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2191                 nid, value, !!(value & NBCFG_ECC_ENABLE));
2192
2193        if (!(value & NBCFG_ECC_ENABLE)) {
2194                amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2195
2196                s->flags.nb_ecc_prev = 0;
2197
2198                /* Attempt to turn on DRAM ECC Enable */
2199                value |= NBCFG_ECC_ENABLE;
2200                amd64_write_pci_cfg(F3, NBCFG, value);
2201
2202                amd64_read_pci_cfg(F3, NBCFG, &value);
2203
2204                if (!(value & NBCFG_ECC_ENABLE)) {
2205                        amd64_warn("Hardware rejected DRAM ECC enable,"
2206                                   "check memory DIMM configuration.\n");
2207                        ret = false;
2208                } else {
2209                        amd64_info("Hardware accepted DRAM ECC Enable\n");
2210                }
2211        } else {
2212                s->flags.nb_ecc_prev = 1;
2213        }
2214
2215        edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2216                 nid, value, !!(value & NBCFG_ECC_ENABLE));
2217
2218        return ret;
2219}
2220
2221static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2222                                        struct pci_dev *F3)
2223{
2224        u32 value, mask = 0x3;          /* UECC/CECC enable */
2225
2226
2227        if (!s->nbctl_valid)
2228                return;
2229
2230        amd64_read_pci_cfg(F3, NBCTL, &value);
2231        value &= ~mask;
2232        value |= s->old_nbctl;
2233
2234        amd64_write_pci_cfg(F3, NBCTL, value);
2235
2236        /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2237        if (!s->flags.nb_ecc_prev) {
2238                amd64_read_pci_cfg(F3, NBCFG, &value);
2239                value &= ~NBCFG_ECC_ENABLE;
2240                amd64_write_pci_cfg(F3, NBCFG, value);
2241        }
2242
2243        /* restore the NB Enable MCGCTL bit */
2244        if (toggle_ecc_err_reporting(s, nid, OFF))
2245                amd64_warn("Error restoring NB MCGCTL settings!\n");
2246}
2247
2248/*
2249 * EDAC requires that the BIOS have ECC enabled before
2250 * taking over the processing of ECC errors. A command line
2251 * option allows to force-enable hardware ECC later in
2252 * enable_ecc_error_reporting().
2253 */
2254static const char *ecc_msg =
2255        "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2256        " Either enable ECC checking or force module loading by setting "
2257        "'ecc_enable_override'.\n"
2258        " (Note that use of the override may cause unknown side effects.)\n";
2259
2260static bool ecc_enabled(struct pci_dev *F3, u16 nid)
2261{
2262        u32 value;
2263        u8 ecc_en = 0;
2264        bool nb_mce_en = false;
2265
2266        amd64_read_pci_cfg(F3, NBCFG, &value);
2267
2268        ecc_en = !!(value & NBCFG_ECC_ENABLE);
2269        amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2270
2271        nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2272        if (!nb_mce_en)
2273                amd64_notice("NB MCE bank disabled, set MSR "
2274                             "0x%08x[4] on node %d to enable.\n",
2275                             MSR_IA32_MCG_CTL, nid);
2276
2277        if (!ecc_en || !nb_mce_en) {
2278                amd64_notice("%s", ecc_msg);
2279                return false;
2280        }
2281        return true;
2282}
2283
2284static int set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2285{
2286        int rc;
2287
2288        rc = amd64_create_sysfs_dbg_files(mci);
2289        if (rc < 0)
2290                return rc;
2291
2292        if (boot_cpu_data.x86 >= 0x10) {
2293                rc = amd64_create_sysfs_inject_files(mci);
2294                if (rc < 0)
2295                        return rc;
2296        }
2297
2298        return 0;
2299}
2300
2301static void del_mc_sysfs_attrs(struct mem_ctl_info *mci)
2302{
2303        amd64_remove_sysfs_dbg_files(mci);
2304
2305        if (boot_cpu_data.x86 >= 0x10)
2306                amd64_remove_sysfs_inject_files(mci);
2307}
2308
2309static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2310                                 struct amd64_family_type *fam)
2311{
2312        struct amd64_pvt *pvt = mci->pvt_info;
2313
2314        mci->mtype_cap          = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2315        mci->edac_ctl_cap       = EDAC_FLAG_NONE;
2316
2317        if (pvt->nbcap & NBCAP_SECDED)
2318                mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2319
2320        if (pvt->nbcap & NBCAP_CHIPKILL)
2321                mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2322
2323        mci->edac_cap           = amd64_determine_edac_cap(pvt);
2324        mci->mod_name           = EDAC_MOD_STR;
2325        mci->mod_ver            = EDAC_AMD64_VERSION;
2326        mci->ctl_name           = fam->ctl_name;
2327        mci->dev_name           = pci_name(pvt->F2);
2328        mci->ctl_page_to_phys   = NULL;
2329
2330        /* memory scrubber interface */
2331        mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2332        mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2333}
2334
2335/*
2336 * returns a pointer to the family descriptor on success, NULL otherwise.
2337 */
2338static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2339{
2340        u8 fam = boot_cpu_data.x86;
2341        struct amd64_family_type *fam_type = NULL;
2342
2343        switch (fam) {
2344        case 0xf:
2345                fam_type                = &amd64_family_types[K8_CPUS];
2346                pvt->ops                = &amd64_family_types[K8_CPUS].ops;
2347                break;
2348
2349        case 0x10:
2350                fam_type                = &amd64_family_types[F10_CPUS];
2351                pvt->ops                = &amd64_family_types[F10_CPUS].ops;
2352                break;
2353
2354        case 0x15:
2355                fam_type                = &amd64_family_types[F15_CPUS];
2356                pvt->ops                = &amd64_family_types[F15_CPUS].ops;
2357                break;
2358
2359        default:
2360                amd64_err("Unsupported family!\n");
2361                return NULL;
2362        }
2363
2364        pvt->ext_model = boot_cpu_data.x86_model >> 4;
2365
2366        amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2367                     (fam == 0xf ?
2368                                (pvt->ext_model >= K8_REV_F  ? "revF or later "
2369                                                             : "revE or earlier ")
2370                                 : ""), pvt->mc_node_id);
2371        return fam_type;
2372}
2373
2374static int amd64_init_one_instance(struct pci_dev *F2)
2375{
2376        struct amd64_pvt *pvt = NULL;
2377        struct amd64_family_type *fam_type = NULL;
2378        struct mem_ctl_info *mci = NULL;
2379        struct edac_mc_layer layers[2];
2380        int err = 0, ret;
2381        u16 nid = amd_get_node_id(F2);
2382
2383        ret = -ENOMEM;
2384        pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2385        if (!pvt)
2386                goto err_ret;
2387
2388        pvt->mc_node_id = nid;
2389        pvt->F2 = F2;
2390
2391        ret = -EINVAL;
2392        fam_type = amd64_per_family_init(pvt);
2393        if (!fam_type)
2394                goto err_free;
2395
2396        ret = -ENODEV;
2397        err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2398        if (err)
2399                goto err_free;
2400
2401        read_mc_regs(pvt);
2402
2403        /*
2404         * We need to determine how many memory channels there are. Then use
2405         * that information for calculating the size of the dynamic instance
2406         * tables in the 'mci' structure.
2407         */
2408        ret = -EINVAL;
2409        pvt->channel_count = pvt->ops->early_channel_count(pvt);
2410        if (pvt->channel_count < 0)
2411                goto err_siblings;
2412
2413        ret = -ENOMEM;
2414        layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2415        layers[0].size = pvt->csels[0].b_cnt;
2416        layers[0].is_virt_csrow = true;
2417        layers[1].type = EDAC_MC_LAYER_CHANNEL;
2418        layers[1].size = pvt->channel_count;
2419        layers[1].is_virt_csrow = false;
2420        mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
2421        if (!mci)
2422                goto err_siblings;
2423
2424        mci->pvt_info = pvt;
2425        mci->pdev = &pvt->F2->dev;
2426
2427        setup_mci_misc_attrs(mci, fam_type);
2428
2429        if (init_csrows(mci))
2430                mci->edac_cap = EDAC_FLAG_NONE;
2431
2432        ret = -ENODEV;
2433        if (edac_mc_add_mc(mci)) {
2434                edac_dbg(1, "failed edac_mc_add_mc()\n");
2435                goto err_add_mc;
2436        }
2437        if (set_mc_sysfs_attrs(mci)) {
2438                edac_dbg(1, "failed edac_mc_add_mc()\n");
2439                goto err_add_sysfs;
2440        }
2441
2442        /* register stuff with EDAC MCE */
2443        if (report_gart_errors)
2444                amd_report_gart_errors(true);
2445
2446        amd_register_ecc_decoder(amd64_decode_bus_error);
2447
2448        mcis[nid] = mci;
2449
2450        atomic_inc(&drv_instances);
2451
2452        return 0;
2453
2454err_add_sysfs:
2455        edac_mc_del_mc(mci->pdev);
2456err_add_mc:
2457        edac_mc_free(mci);
2458
2459err_siblings:
2460        free_mc_sibling_devs(pvt);
2461
2462err_free:
2463        kfree(pvt);
2464
2465err_ret:
2466        return ret;
2467}
2468
2469static int amd64_probe_one_instance(struct pci_dev *pdev,
2470                                    const struct pci_device_id *mc_type)
2471{
2472        u16 nid = amd_get_node_id(pdev);
2473        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2474        struct ecc_settings *s;
2475        int ret = 0;
2476
2477        ret = pci_enable_device(pdev);
2478        if (ret < 0) {
2479                edac_dbg(0, "ret=%d\n", ret);
2480                return -EIO;
2481        }
2482
2483        ret = -ENOMEM;
2484        s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2485        if (!s)
2486                goto err_out;
2487
2488        ecc_stngs[nid] = s;
2489
2490        if (!ecc_enabled(F3, nid)) {
2491                ret = -ENODEV;
2492
2493                if (!ecc_enable_override)
2494                        goto err_enable;
2495
2496                amd64_warn("Forcing ECC on!\n");
2497
2498                if (!enable_ecc_error_reporting(s, nid, F3))
2499                        goto err_enable;
2500        }
2501
2502        ret = amd64_init_one_instance(pdev);
2503        if (ret < 0) {
2504                amd64_err("Error probing instance: %d\n", nid);
2505                restore_ecc_error_reporting(s, nid, F3);
2506        }
2507
2508        return ret;
2509
2510err_enable:
2511        kfree(s);
2512        ecc_stngs[nid] = NULL;
2513
2514err_out:
2515        return ret;
2516}
2517
2518static void amd64_remove_one_instance(struct pci_dev *pdev)
2519{
2520        struct mem_ctl_info *mci;
2521        struct amd64_pvt *pvt;
2522        u16 nid = amd_get_node_id(pdev);
2523        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2524        struct ecc_settings *s = ecc_stngs[nid];
2525
2526        mci = find_mci_by_dev(&pdev->dev);
2527        del_mc_sysfs_attrs(mci);
2528        /* Remove from EDAC CORE tracking list */
2529        mci = edac_mc_del_mc(&pdev->dev);
2530        if (!mci)
2531                return;
2532
2533        pvt = mci->pvt_info;
2534
2535        restore_ecc_error_reporting(s, nid, F3);
2536
2537        free_mc_sibling_devs(pvt);
2538
2539        /* unregister from EDAC MCE */
2540        amd_report_gart_errors(false);
2541        amd_unregister_ecc_decoder(amd64_decode_bus_error);
2542
2543        kfree(ecc_stngs[nid]);
2544        ecc_stngs[nid] = NULL;
2545
2546        /* Free the EDAC CORE resources */
2547        mci->pvt_info = NULL;
2548        mcis[nid] = NULL;
2549
2550        kfree(pvt);
2551        edac_mc_free(mci);
2552}
2553
2554/*
2555 * This table is part of the interface for loading drivers for PCI devices. The
2556 * PCI core identifies what devices are on a system during boot, and then
2557 * inquiry this table to see if this driver is for a given device found.
2558 */
2559static DEFINE_PCI_DEVICE_TABLE(amd64_pci_table) = {
2560        {
2561                .vendor         = PCI_VENDOR_ID_AMD,
2562                .device         = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2563                .subvendor      = PCI_ANY_ID,
2564                .subdevice      = PCI_ANY_ID,
2565                .class          = 0,
2566                .class_mask     = 0,
2567        },
2568        {
2569                .vendor         = PCI_VENDOR_ID_AMD,
2570                .device         = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2571                .subvendor      = PCI_ANY_ID,
2572                .subdevice      = PCI_ANY_ID,
2573                .class          = 0,
2574                .class_mask     = 0,
2575        },
2576        {
2577                .vendor         = PCI_VENDOR_ID_AMD,
2578                .device         = PCI_DEVICE_ID_AMD_15H_NB_F2,
2579                .subvendor      = PCI_ANY_ID,
2580                .subdevice      = PCI_ANY_ID,
2581                .class          = 0,
2582                .class_mask     = 0,
2583        },
2584
2585        {0, }
2586};
2587MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2588
2589static struct pci_driver amd64_pci_driver = {
2590        .name           = EDAC_MOD_STR,
2591        .probe          = amd64_probe_one_instance,
2592        .remove         = amd64_remove_one_instance,
2593        .id_table       = amd64_pci_table,
2594};
2595
2596static void setup_pci_device(void)
2597{
2598        struct mem_ctl_info *mci;
2599        struct amd64_pvt *pvt;
2600
2601        if (amd64_ctl_pci)
2602                return;
2603
2604        mci = mcis[0];
2605        if (mci) {
2606
2607                pvt = mci->pvt_info;
2608                amd64_ctl_pci =
2609                        edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2610
2611                if (!amd64_ctl_pci) {
2612                        pr_warning("%s(): Unable to create PCI control\n",
2613                                   __func__);
2614
2615                        pr_warning("%s(): PCI error report via EDAC not set\n",
2616                                   __func__);
2617                        }
2618        }
2619}
2620
2621static int __init amd64_edac_init(void)
2622{
2623        int err = -ENODEV;
2624
2625        printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2626
2627        opstate_init();
2628
2629        if (amd_cache_northbridges() < 0)
2630                goto err_ret;
2631
2632        err = -ENOMEM;
2633        mcis      = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2634        ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2635        if (!(mcis && ecc_stngs))
2636                goto err_free;
2637
2638        msrs = msrs_alloc();
2639        if (!msrs)
2640                goto err_free;
2641
2642        err = pci_register_driver(&amd64_pci_driver);
2643        if (err)
2644                goto err_pci;
2645
2646        err = -ENODEV;
2647        if (!atomic_read(&drv_instances))
2648                goto err_no_instances;
2649
2650        setup_pci_device();
2651        return 0;
2652
2653err_no_instances:
2654        pci_unregister_driver(&amd64_pci_driver);
2655
2656err_pci:
2657        msrs_free(msrs);
2658        msrs = NULL;
2659
2660err_free:
2661        kfree(mcis);
2662        mcis = NULL;
2663
2664        kfree(ecc_stngs);
2665        ecc_stngs = NULL;
2666
2667err_ret:
2668        return err;
2669}
2670
2671static void __exit amd64_edac_exit(void)
2672{
2673        if (amd64_ctl_pci)
2674                edac_pci_release_generic_ctl(amd64_ctl_pci);
2675
2676        pci_unregister_driver(&amd64_pci_driver);
2677
2678        kfree(ecc_stngs);
2679        ecc_stngs = NULL;
2680
2681        kfree(mcis);
2682        mcis = NULL;
2683
2684        msrs_free(msrs);
2685        msrs = NULL;
2686}
2687
2688module_init(amd64_edac_init);
2689module_exit(amd64_edac_exit);
2690
2691MODULE_LICENSE("GPL");
2692MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2693                "Dave Peterson, Thayne Harbaugh");
2694MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2695                EDAC_AMD64_VERSION);
2696
2697module_param(edac_op_state, int, 0444);
2698MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
2699
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.