linux/drivers/edac/amd64_edac.c
<<
>>
Prefs
   1#include "amd64_edac.h"
   2#include <asm/amd_nb.h>
   3
   4static struct edac_pci_ctl_info *amd64_ctl_pci;
   5
   6static int report_gart_errors;
   7module_param(report_gart_errors, int, 0644);
   8
   9/*
  10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
  11 * cleared to prevent re-enabling the hardware by this driver.
  12 */
  13static int ecc_enable_override;
  14module_param(ecc_enable_override, int, 0644);
  15
  16static struct msr __percpu *msrs;
  17
  18/*
  19 * count successfully initialized driver instances for setup_pci_device()
  20 */
  21static atomic_t drv_instances = ATOMIC_INIT(0);
  22
  23/* Per-node driver instances */
  24static struct mem_ctl_info **mcis;
  25static struct ecc_settings **ecc_stngs;
  26
  27/*
  28 * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and
  29 * later.
  30 */
  31static int ddr2_dbam_revCG[] = {
  32                           [0]          = 32,
  33                           [1]          = 64,
  34                           [2]          = 128,
  35                           [3]          = 256,
  36                           [4]          = 512,
  37                           [5]          = 1024,
  38                           [6]          = 2048,
  39};
  40
  41static int ddr2_dbam_revD[] = {
  42                           [0]          = 32,
  43                           [1]          = 64,
  44                           [2 ... 3]    = 128,
  45                           [4]          = 256,
  46                           [5]          = 512,
  47                           [6]          = 256,
  48                           [7]          = 512,
  49                           [8 ... 9]    = 1024,
  50                           [10]         = 2048,
  51};
  52
  53static int ddr2_dbam[] = { [0]          = 128,
  54                           [1]          = 256,
  55                           [2 ... 4]    = 512,
  56                           [5 ... 6]    = 1024,
  57                           [7 ... 8]    = 2048,
  58                           [9 ... 10]   = 4096,
  59                           [11]         = 8192,
  60};
  61
  62static int ddr3_dbam[] = { [0]          = -1,
  63                           [1]          = 256,
  64                           [2]          = 512,
  65                           [3 ... 4]    = -1,
  66                           [5 ... 6]    = 1024,
  67                           [7 ... 8]    = 2048,
  68                           [9 ... 10]   = 4096,
  69                           [11]         = 8192,
  70};
  71
  72/*
  73 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
  74 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
  75 * or higher value'.
  76 *
  77 *FIXME: Produce a better mapping/linearisation.
  78 */
  79
  80
  81struct scrubrate {
  82       u32 scrubval;           /* bit pattern for scrub rate */
  83       u32 bandwidth;          /* bandwidth consumed (bytes/sec) */
  84} scrubrates[] = {
  85        { 0x01, 1600000000UL},
  86        { 0x02, 800000000UL},
  87        { 0x03, 400000000UL},
  88        { 0x04, 200000000UL},
  89        { 0x05, 100000000UL},
  90        { 0x06, 50000000UL},
  91        { 0x07, 25000000UL},
  92        { 0x08, 12284069UL},
  93        { 0x09, 6274509UL},
  94        { 0x0A, 3121951UL},
  95        { 0x0B, 1560975UL},
  96        { 0x0C, 781440UL},
  97        { 0x0D, 390720UL},
  98        { 0x0E, 195300UL},
  99        { 0x0F, 97650UL},
 100        { 0x10, 48854UL},
 101        { 0x11, 24427UL},
 102        { 0x12, 12213UL},
 103        { 0x13, 6101UL},
 104        { 0x14, 3051UL},
 105        { 0x15, 1523UL},
 106        { 0x16, 761UL},
 107        { 0x00, 0UL},        /* scrubbing off */
 108};
 109
 110/*
 111 * Memory scrubber control interface. For K8, memory scrubbing is handled by
 112 * hardware and can involve L2 cache, dcache as well as the main memory. With
 113 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
 114 * functionality.
 115 *
 116 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
 117 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
 118 * bytes/sec for the setting.
 119 *
 120 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
 121 * other archs, we might not have access to the caches directly.
 122 */
 123
 124/*
 125 * scan the scrub rate mapping table for a close or matching bandwidth value to
 126 * issue. If requested is too big, then use last maximum value found.
 127 */
 128static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
 129{
 130        u32 scrubval;
 131        int i;
 132
 133        /*
 134         * map the configured rate (new_bw) to a value specific to the AMD64
 135         * memory controller and apply to register. Search for the first
 136         * bandwidth entry that is greater or equal than the setting requested
 137         * and program that. If at last entry, turn off DRAM scrubbing.
 138         */
 139        for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
 140                /*
 141                 * skip scrub rates which aren't recommended
 142                 * (see F10 BKDG, F3x58)
 143                 */
 144                if (scrubrates[i].scrubval < min_rate)
 145                        continue;
 146
 147                if (scrubrates[i].bandwidth <= new_bw)
 148                        break;
 149
 150                /*
 151                 * if no suitable bandwidth found, turn off DRAM scrubbing
 152                 * entirely by falling back to the last element in the
 153                 * scrubrates array.
 154                 */
 155        }
 156
 157        scrubval = scrubrates[i].scrubval;
 158
 159        pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F);
 160
 161        if (scrubval)
 162                return scrubrates[i].bandwidth;
 163
 164        return 0;
 165}
 166
 167static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
 168{
 169        struct amd64_pvt *pvt = mci->pvt_info;
 170
 171        return __amd64_set_scrub_rate(pvt->F3, bw, pvt->min_scrubrate);
 172}
 173
 174static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
 175{
 176        struct amd64_pvt *pvt = mci->pvt_info;
 177        u32 scrubval = 0;
 178        int i, retval = -EINVAL;
 179
 180        amd64_read_pci_cfg(pvt->F3, K8_SCRCTRL, &scrubval);
 181
 182        scrubval = scrubval & 0x001F;
 183
 184        amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval);
 185
 186        for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
 187                if (scrubrates[i].scrubval == scrubval) {
 188                        retval = scrubrates[i].bandwidth;
 189                        break;
 190                }
 191        }
 192        return retval;
 193}
 194
 195/* Map from a CSROW entry to the mask entry that operates on it */
 196static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow)
 197{
 198        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F)
 199                return csrow;
 200        else
 201                return csrow >> 1;
 202}
 203
 204/* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */
 205static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow)
 206{
 207        if (dct == 0)
 208                return pvt->dcsb0[csrow];
 209        else
 210                return pvt->dcsb1[csrow];
 211}
 212
 213/*
 214 * Return the 'mask' address the i'th CS entry. This function is needed because
 215 * there number of DCSM registers on Rev E and prior vs Rev F and later is
 216 * different.
 217 */
 218static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow)
 219{
 220        if (dct == 0)
 221                return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)];
 222        else
 223                return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)];
 224}
 225
 226
 227/*
 228 * In *base and *limit, pass back the full 40-bit base and limit physical
 229 * addresses for the node given by node_id.  This information is obtained from
 230 * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The
 231 * base and limit addresses are of type SysAddr, as defined at the start of
 232 * section 3.4.4 (p. 70).  They are the lowest and highest physical addresses
 233 * in the address range they represent.
 234 */
 235static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id,
 236                               u64 *base, u64 *limit)
 237{
 238        *base = pvt->dram_base[node_id];
 239        *limit = pvt->dram_limit[node_id];
 240}
 241
 242/*
 243 * Return 1 if the SysAddr given by sys_addr matches the base/limit associated
 244 * with node_id
 245 */
 246static int amd64_base_limit_match(struct amd64_pvt *pvt,
 247                                        u64 sys_addr, int node_id)
 248{
 249        u64 base, limit, addr;
 250
 251        amd64_get_base_and_limit(pvt, node_id, &base, &limit);
 252
 253        /* The K8 treats this as a 40-bit value.  However, bits 63-40 will be
 254         * all ones if the most significant implemented address bit is 1.
 255         * Here we discard bits 63-40.  See section 3.4.2 of AMD publication
 256         * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
 257         * Application Programming.
 258         */
 259        addr = sys_addr & 0x000000ffffffffffull;
 260
 261        return (addr >= base) && (addr <= limit);
 262}
 263
 264/*
 265 * Attempt to map a SysAddr to a node. On success, return a pointer to the
 266 * mem_ctl_info structure for the node that the SysAddr maps to.
 267 *
 268 * On failure, return NULL.
 269 */
 270static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
 271                                                u64 sys_addr)
 272{
 273        struct amd64_pvt *pvt;
 274        int node_id;
 275        u32 intlv_en, bits;
 276
 277        /*
 278         * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
 279         * 3.4.4.2) registers to map the SysAddr to a node ID.
 280         */
 281        pvt = mci->pvt_info;
 282
 283        /*
 284         * The value of this field should be the same for all DRAM Base
 285         * registers.  Therefore we arbitrarily choose to read it from the
 286         * register for node 0.
 287         */
 288        intlv_en = pvt->dram_IntlvEn[0];
 289
 290        if (intlv_en == 0) {
 291                for (node_id = 0; node_id < DRAM_REG_COUNT; node_id++) {
 292                        if (amd64_base_limit_match(pvt, sys_addr, node_id))
 293                                goto found;
 294                }
 295                goto err_no_match;
 296        }
 297
 298        if (unlikely((intlv_en != 0x01) &&
 299                     (intlv_en != 0x03) &&
 300                     (intlv_en != 0x07))) {
 301                amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
 302                return NULL;
 303        }
 304
 305        bits = (((u32) sys_addr) >> 12) & intlv_en;
 306
 307        for (node_id = 0; ; ) {
 308                if ((pvt->dram_IntlvSel[node_id] & intlv_en) == bits)
 309                        break;  /* intlv_sel field matches */
 310
 311                if (++node_id >= DRAM_REG_COUNT)
 312                        goto err_no_match;
 313        }
 314
 315        /* sanity test for sys_addr */
 316        if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
 317                amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
 318                           "range for node %d with node interleaving enabled.\n",
 319                           __func__, sys_addr, node_id);
 320                return NULL;
 321        }
 322
 323found:
 324        return edac_mc_find(node_id);
 325
 326err_no_match:
 327        debugf2("sys_addr 0x%lx doesn't match any node\n",
 328                (unsigned long)sys_addr);
 329
 330        return NULL;
 331}
 332
 333/*
 334 * Extract the DRAM CS base address from selected csrow register.
 335 */
 336static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow)
 337{
 338        return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) <<
 339                                pvt->dcs_shift;
 340}
 341
 342/*
 343 * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way.
 344 */
 345static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow)
 346{
 347        u64 dcsm_bits, other_bits;
 348        u64 mask;
 349
 350        /* Extract bits from DRAM CS Mask. */
 351        dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask;
 352
 353        other_bits = pvt->dcsm_mask;
 354        other_bits = ~(other_bits << pvt->dcs_shift);
 355
 356        /*
 357         * The extracted bits from DCSM belong in the spaces represented by
 358         * the cleared bits in other_bits.
 359         */
 360        mask = (dcsm_bits << pvt->dcs_shift) | other_bits;
 361
 362        return mask;
 363}
 364
 365/*
 366 * @input_addr is an InputAddr associated with the node given by mci. Return the
 367 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
 368 */
 369static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
 370{
 371        struct amd64_pvt *pvt;
 372        int csrow;
 373        u64 base, mask;
 374
 375        pvt = mci->pvt_info;
 376
 377        /*
 378         * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS
 379         * base/mask register pair, test the condition shown near the start of
 380         * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E).
 381         */
 382        for (csrow = 0; csrow < pvt->cs_count; csrow++) {
 383
 384                /* This DRAM chip select is disabled on this node */
 385                if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0)
 386                        continue;
 387
 388                base = base_from_dct_base(pvt, csrow);
 389                mask = ~mask_from_dct_mask(pvt, csrow);
 390
 391                if ((input_addr & mask) == (base & mask)) {
 392                        debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
 393                                (unsigned long)input_addr, csrow,
 394                                pvt->mc_node_id);
 395
 396                        return csrow;
 397                }
 398        }
 399
 400        debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
 401                (unsigned long)input_addr, pvt->mc_node_id);
 402
 403        return -1;
 404}
 405
 406/*
 407 * Return the base value defined by the DRAM Base register for the node
 408 * represented by mci.  This function returns the full 40-bit value despite the
 409 * fact that the register only stores bits 39-24 of the value. See section
 410 * 3.4.4.1 (BKDG #26094, K8, revA-E)
 411 */
 412static inline u64 get_dram_base(struct mem_ctl_info *mci)
 413{
 414        struct amd64_pvt *pvt = mci->pvt_info;
 415
 416        return pvt->dram_base[pvt->mc_node_id];
 417}
 418
 419/*
 420 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
 421 * for the node represented by mci. Info is passed back in *hole_base,
 422 * *hole_offset, and *hole_size.  Function returns 0 if info is valid or 1 if
 423 * info is invalid. Info may be invalid for either of the following reasons:
 424 *
 425 * - The revision of the node is not E or greater.  In this case, the DRAM Hole
 426 *   Address Register does not exist.
 427 *
 428 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
 429 *   indicating that its contents are not valid.
 430 *
 431 * The values passed back in *hole_base, *hole_offset, and *hole_size are
 432 * complete 32-bit values despite the fact that the bitfields in the DHAR
 433 * only represent bits 31-24 of the base and offset values.
 434 */
 435int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
 436                             u64 *hole_offset, u64 *hole_size)
 437{
 438        struct amd64_pvt *pvt = mci->pvt_info;
 439        u64 base;
 440
 441        /* only revE and later have the DRAM Hole Address Register */
 442        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
 443                debugf1("  revision %d for node %d does not support DHAR\n",
 444                        pvt->ext_model, pvt->mc_node_id);
 445                return 1;
 446        }
 447
 448        /* only valid for Fam10h */
 449        if (boot_cpu_data.x86 == 0x10 &&
 450            (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) {
 451                debugf1("  Dram Memory Hoisting is DISABLED on this system\n");
 452                return 1;
 453        }
 454
 455        if ((pvt->dhar & DHAR_VALID) == 0) {
 456                debugf1("  Dram Memory Hoisting is DISABLED on this node %d\n",
 457                        pvt->mc_node_id);
 458                return 1;
 459        }
 460
 461        /* This node has Memory Hoisting */
 462
 463        /* +------------------+--------------------+--------------------+-----
 464         * | memory           | DRAM hole          | relocated          |
 465         * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
 466         * |                  |                    | DRAM hole          |
 467         * |                  |                    | [0x100000000,      |
 468         * |                  |                    |  (0x100000000+     |
 469         * |                  |                    |   (0xffffffff-x))] |
 470         * +------------------+--------------------+--------------------+-----
 471         *
 472         * Above is a diagram of physical memory showing the DRAM hole and the
 473         * relocated addresses from the DRAM hole.  As shown, the DRAM hole
 474         * starts at address x (the base address) and extends through address
 475         * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
 476         * addresses in the hole so that they start at 0x100000000.
 477         */
 478
 479        base = dhar_base(pvt->dhar);
 480
 481        *hole_base = base;
 482        *hole_size = (0x1ull << 32) - base;
 483
 484        if (boot_cpu_data.x86 > 0xf)
 485                *hole_offset = f10_dhar_offset(pvt->dhar);
 486        else
 487                *hole_offset = k8_dhar_offset(pvt->dhar);
 488
 489        debugf1("  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
 490                pvt->mc_node_id, (unsigned long)*hole_base,
 491                (unsigned long)*hole_offset, (unsigned long)*hole_size);
 492
 493        return 0;
 494}
 495EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
 496
 497/*
 498 * Return the DramAddr that the SysAddr given by @sys_addr maps to.  It is
 499 * assumed that sys_addr maps to the node given by mci.
 500 *
 501 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
 502 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
 503 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
 504 * then it is also involved in translating a SysAddr to a DramAddr. Sections
 505 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
 506 * These parts of the documentation are unclear. I interpret them as follows:
 507 *
 508 * When node n receives a SysAddr, it processes the SysAddr as follows:
 509 *
 510 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
 511 *    Limit registers for node n. If the SysAddr is not within the range
 512 *    specified by the base and limit values, then node n ignores the Sysaddr
 513 *    (since it does not map to node n). Otherwise continue to step 2 below.
 514 *
 515 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
 516 *    disabled so skip to step 3 below. Otherwise see if the SysAddr is within
 517 *    the range of relocated addresses (starting at 0x100000000) from the DRAM
 518 *    hole. If not, skip to step 3 below. Else get the value of the
 519 *    DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
 520 *    offset defined by this value from the SysAddr.
 521 *
 522 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
 523 *    Base register for node n. To obtain the DramAddr, subtract the base
 524 *    address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
 525 */
 526static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
 527{
 528        u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
 529        int ret = 0;
 530
 531        dram_base = get_dram_base(mci);
 532
 533        ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
 534                                      &hole_size);
 535        if (!ret) {
 536                if ((sys_addr >= (1ull << 32)) &&
 537                    (sys_addr < ((1ull << 32) + hole_size))) {
 538                        /* use DHAR to translate SysAddr to DramAddr */
 539                        dram_addr = sys_addr - hole_offset;
 540
 541                        debugf2("using DHAR to translate SysAddr 0x%lx to "
 542                                "DramAddr 0x%lx\n",
 543                                (unsigned long)sys_addr,
 544                                (unsigned long)dram_addr);
 545
 546                        return dram_addr;
 547                }
 548        }
 549
 550        /*
 551         * Translate the SysAddr to a DramAddr as shown near the start of
 552         * section 3.4.4 (p. 70).  Although sys_addr is a 64-bit value, the k8
 553         * only deals with 40-bit values.  Therefore we discard bits 63-40 of
 554         * sys_addr below.  If bit 39 of sys_addr is 1 then the bits we
 555         * discard are all 1s.  Otherwise the bits we discard are all 0s.  See
 556         * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
 557         * Programmer's Manual Volume 1 Application Programming.
 558         */
 559        dram_addr = (sys_addr & 0xffffffffffull) - dram_base;
 560
 561        debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
 562                "DramAddr 0x%lx\n", (unsigned long)sys_addr,
 563                (unsigned long)dram_addr);
 564        return dram_addr;
 565}
 566
 567/*
 568 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
 569 * (section 3.4.4.1).  Return the number of bits from a SysAddr that are used
 570 * for node interleaving.
 571 */
 572static int num_node_interleave_bits(unsigned intlv_en)
 573{
 574        static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
 575        int n;
 576
 577        BUG_ON(intlv_en > 7);
 578        n = intlv_shift_table[intlv_en];
 579        return n;
 580}
 581
 582/* Translate the DramAddr given by @dram_addr to an InputAddr. */
 583static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
 584{
 585        struct amd64_pvt *pvt;
 586        int intlv_shift;
 587        u64 input_addr;
 588
 589        pvt = mci->pvt_info;
 590
 591        /*
 592         * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
 593         * concerning translating a DramAddr to an InputAddr.
 594         */
 595        intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]);
 596        input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) +
 597            (dram_addr & 0xfff);
 598
 599        debugf2("  Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
 600                intlv_shift, (unsigned long)dram_addr,
 601                (unsigned long)input_addr);
 602
 603        return input_addr;
 604}
 605
 606/*
 607 * Translate the SysAddr represented by @sys_addr to an InputAddr.  It is
 608 * assumed that @sys_addr maps to the node given by mci.
 609 */
 610static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
 611{
 612        u64 input_addr;
 613
 614        input_addr =
 615            dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
 616
 617        debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
 618                (unsigned long)sys_addr, (unsigned long)input_addr);
 619
 620        return input_addr;
 621}
 622
 623
 624/*
 625 * @input_addr is an InputAddr associated with the node represented by mci.
 626 * Translate @input_addr to a DramAddr and return the result.
 627 */
 628static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
 629{
 630        struct amd64_pvt *pvt;
 631        int node_id, intlv_shift;
 632        u64 bits, dram_addr;
 633        u32 intlv_sel;
 634
 635        /*
 636         * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
 637         * shows how to translate a DramAddr to an InputAddr. Here we reverse
 638         * this procedure. When translating from a DramAddr to an InputAddr, the
 639         * bits used for node interleaving are discarded.  Here we recover these
 640         * bits from the IntlvSel field of the DRAM Limit register (section
 641         * 3.4.4.2) for the node that input_addr is associated with.
 642         */
 643        pvt = mci->pvt_info;
 644        node_id = pvt->mc_node_id;
 645        BUG_ON((node_id < 0) || (node_id > 7));
 646
 647        intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]);
 648
 649        if (intlv_shift == 0) {
 650                debugf1("    InputAddr 0x%lx translates to DramAddr of "
 651                        "same value\n", (unsigned long)input_addr);
 652
 653                return input_addr;
 654        }
 655
 656        bits = ((input_addr & 0xffffff000ull) << intlv_shift) +
 657            (input_addr & 0xfff);
 658
 659        intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1);
 660        dram_addr = bits + (intlv_sel << 12);
 661
 662        debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
 663                "(%d node interleave bits)\n", (unsigned long)input_addr,
 664                (unsigned long)dram_addr, intlv_shift);
 665
 666        return dram_addr;
 667}
 668
 669/*
 670 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
 671 * @dram_addr to a SysAddr.
 672 */
 673static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
 674{
 675        struct amd64_pvt *pvt = mci->pvt_info;
 676        u64 hole_base, hole_offset, hole_size, base, limit, sys_addr;
 677        int ret = 0;
 678
 679        ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
 680                                      &hole_size);
 681        if (!ret) {
 682                if ((dram_addr >= hole_base) &&
 683                    (dram_addr < (hole_base + hole_size))) {
 684                        sys_addr = dram_addr + hole_offset;
 685
 686                        debugf1("using DHAR to translate DramAddr 0x%lx to "
 687                                "SysAddr 0x%lx\n", (unsigned long)dram_addr,
 688                                (unsigned long)sys_addr);
 689
 690                        return sys_addr;
 691                }
 692        }
 693
 694        amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit);
 695        sys_addr = dram_addr + base;
 696
 697        /*
 698         * The sys_addr we have computed up to this point is a 40-bit value
 699         * because the k8 deals with 40-bit values.  However, the value we are
 700         * supposed to return is a full 64-bit physical address.  The AMD
 701         * x86-64 architecture specifies that the most significant implemented
 702         * address bit through bit 63 of a physical address must be either all
 703         * 0s or all 1s.  Therefore we sign-extend the 40-bit sys_addr to a
 704         * 64-bit value below.  See section 3.4.2 of AMD publication 24592:
 705         * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
 706         * Programming.
 707         */
 708        sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
 709
 710        debugf1("    Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
 711                pvt->mc_node_id, (unsigned long)dram_addr,
 712                (unsigned long)sys_addr);
 713
 714        return sys_addr;
 715}
 716
 717/*
 718 * @input_addr is an InputAddr associated with the node given by mci. Translate
 719 * @input_addr to a SysAddr.
 720 */
 721static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
 722                                         u64 input_addr)
 723{
 724        return dram_addr_to_sys_addr(mci,
 725                                     input_addr_to_dram_addr(mci, input_addr));
 726}
 727
 728/*
 729 * Find the minimum and maximum InputAddr values that map to the given @csrow.
 730 * Pass back these values in *input_addr_min and *input_addr_max.
 731 */
 732static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
 733                              u64 *input_addr_min, u64 *input_addr_max)
 734{
 735        struct amd64_pvt *pvt;
 736        u64 base, mask;
 737
 738        pvt = mci->pvt_info;
 739        BUG_ON((csrow < 0) || (csrow >= pvt->cs_count));
 740
 741        base = base_from_dct_base(pvt, csrow);
 742        mask = mask_from_dct_mask(pvt, csrow);
 743
 744        *input_addr_min = base & ~mask;
 745        *input_addr_max = base | mask | pvt->dcs_mask_notused;
 746}
 747
 748/* Map the Error address to a PAGE and PAGE OFFSET. */
 749static inline void error_address_to_page_and_offset(u64 error_address,
 750                                                    u32 *page, u32 *offset)
 751{
 752        *page = (u32) (error_address >> PAGE_SHIFT);
 753        *offset = ((u32) error_address) & ~PAGE_MASK;
 754}
 755
 756/*
 757 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
 758 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
 759 * of a node that detected an ECC memory error.  mci represents the node that
 760 * the error address maps to (possibly different from the node that detected
 761 * the error).  Return the number of the csrow that sys_addr maps to, or -1 on
 762 * error.
 763 */
 764static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
 765{
 766        int csrow;
 767
 768        csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
 769
 770        if (csrow == -1)
 771                amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
 772                                  "address 0x%lx\n", (unsigned long)sys_addr);
 773        return csrow;
 774}
 775
 776static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
 777
 778static u16 extract_syndrome(struct err_regs *err)
 779{
 780        return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00);
 781}
 782
 783/*
 784 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
 785 * are ECC capable.
 786 */
 787static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
 788{
 789        int bit;
 790        enum dev_type edac_cap = EDAC_FLAG_NONE;
 791
 792        bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
 793                ? 19
 794                : 17;
 795
 796        if (pvt->dclr0 & BIT(bit))
 797                edac_cap = EDAC_FLAG_SECDED;
 798
 799        return edac_cap;
 800}
 801
 802
 803static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt);
 804
 805static void amd64_dump_dramcfg_low(u32 dclr, int chan)
 806{
 807        debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
 808
 809        debugf1("  DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
 810                (dclr & BIT(16)) ?  "un" : "",
 811                (dclr & BIT(19)) ? "yes" : "no");
 812
 813        debugf1("  PAR/ERR parity: %s\n",
 814                (dclr & BIT(8)) ?  "enabled" : "disabled");
 815
 816        debugf1("  DCT 128bit mode width: %s\n",
 817                (dclr & BIT(11)) ?  "128b" : "64b");
 818
 819        debugf1("  x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
 820                (dclr & BIT(12)) ?  "yes" : "no",
 821                (dclr & BIT(13)) ?  "yes" : "no",
 822                (dclr & BIT(14)) ?  "yes" : "no",
 823                (dclr & BIT(15)) ?  "yes" : "no");
 824}
 825
 826/* Display and decode various NB registers for debug purposes. */
 827static void amd64_dump_misc_regs(struct amd64_pvt *pvt)
 828{
 829        debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
 830
 831        debugf1("  NB two channel DRAM capable: %s\n",
 832                (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no");
 833
 834        debugf1("  ECC capable: %s, ChipKill ECC capable: %s\n",
 835                (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no",
 836                (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no");
 837
 838        amd64_dump_dramcfg_low(pvt->dclr0, 0);
 839
 840        debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
 841
 842        debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
 843                        "offset: 0x%08x\n",
 844                        pvt->dhar,
 845                        dhar_base(pvt->dhar),
 846                        (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt->dhar)
 847                                                   : f10_dhar_offset(pvt->dhar));
 848
 849        debugf1("  DramHoleValid: %s\n",
 850                (pvt->dhar & DHAR_VALID) ? "yes" : "no");
 851
 852        amd64_debug_display_dimm_sizes(0, pvt);
 853
 854        /* everything below this point is Fam10h and above */
 855        if (boot_cpu_data.x86 == 0xf)
 856                return;
 857
 858        amd64_debug_display_dimm_sizes(1, pvt);
 859
 860        amd64_info("using %s syndromes.\n", ((pvt->syn_type == 8) ? "x8" : "x4"));
 861
 862        /* Only if NOT ganged does dclr1 have valid info */
 863        if (!dct_ganging_enabled(pvt))
 864                amd64_dump_dramcfg_low(pvt->dclr1, 1);
 865}
 866
 867/* Read in both of DBAM registers */
 868static void amd64_read_dbam_reg(struct amd64_pvt *pvt)
 869{
 870        amd64_read_pci_cfg(pvt->F2, DBAM0, &pvt->dbam0);
 871
 872        if (boot_cpu_data.x86 >= 0x10)
 873                amd64_read_pci_cfg(pvt->F2, DBAM1, &pvt->dbam1);
 874}
 875
 876/*
 877 * NOTE: CPU Revision Dependent code: Rev E and Rev F
 878 *
 879 * Set the DCSB and DCSM mask values depending on the CPU revision value. Also
 880 * set the shift factor for the DCSB and DCSM values.
 881 *
 882 * ->dcs_mask_notused, RevE:
 883 *
 884 * To find the max InputAddr for the csrow, start with the base address and set
 885 * all bits that are "don't care" bits in the test at the start of section
 886 * 3.5.4 (p. 84).
 887 *
 888 * The "don't care" bits are all set bits in the mask and all bits in the gaps
 889 * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS
 890 * represents bits [24:20] and [12:0], which are all bits in the above-mentioned
 891 * gaps.
 892 *
 893 * ->dcs_mask_notused, RevF and later:
 894 *
 895 * To find the max InputAddr for the csrow, start with the base address and set
 896 * all bits that are "don't care" bits in the test at the start of NPT section
 897 * 4.5.4 (p. 87).
 898 *
 899 * The "don't care" bits are all set bits in the mask and all bits in the gaps
 900 * between bit ranges [36:27] and [21:13].
 901 *
 902 * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0],
 903 * which are all bits in the above-mentioned gaps.
 904 */
 905static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt)
 906{
 907
 908        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
 909                pvt->dcsb_base          = REV_E_DCSB_BASE_BITS;
 910                pvt->dcsm_mask          = REV_E_DCSM_MASK_BITS;
 911                pvt->dcs_mask_notused   = REV_E_DCS_NOTUSED_BITS;
 912                pvt->dcs_shift          = REV_E_DCS_SHIFT;
 913                pvt->cs_count           = 8;
 914                pvt->num_dcsm           = 8;
 915        } else {
 916                pvt->dcsb_base          = REV_F_F1Xh_DCSB_BASE_BITS;
 917                pvt->dcsm_mask          = REV_F_F1Xh_DCSM_MASK_BITS;
 918                pvt->dcs_mask_notused   = REV_F_F1Xh_DCS_NOTUSED_BITS;
 919                pvt->dcs_shift          = REV_F_F1Xh_DCS_SHIFT;
 920                pvt->cs_count           = 8;
 921                pvt->num_dcsm           = 4;
 922        }
 923}
 924
 925/*
 926 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers
 927 */
 928static void amd64_read_dct_base_mask(struct amd64_pvt *pvt)
 929{
 930        int cs, reg;
 931
 932        amd64_set_dct_base_and_mask(pvt);
 933
 934        for (cs = 0; cs < pvt->cs_count; cs++) {
 935                reg = K8_DCSB0 + (cs * 4);
 936                if (!amd64_read_pci_cfg(pvt->F2, reg, &pvt->dcsb0[cs]))
 937                        debugf0("  DCSB0[%d]=0x%08x reg: F2x%x\n",
 938                                cs, pvt->dcsb0[cs], reg);
 939
 940                /* If DCT are NOT ganged, then read in DCT1's base */
 941                if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) {
 942                        reg = F10_DCSB1 + (cs * 4);
 943                        if (!amd64_read_pci_cfg(pvt->F2, reg,
 944                                                &pvt->dcsb1[cs]))
 945                                debugf0("  DCSB1[%d]=0x%08x reg: F2x%x\n",
 946                                        cs, pvt->dcsb1[cs], reg);
 947                } else {
 948                        pvt->dcsb1[cs] = 0;
 949                }
 950        }
 951
 952        for (cs = 0; cs < pvt->num_dcsm; cs++) {
 953                reg = K8_DCSM0 + (cs * 4);
 954                if (!amd64_read_pci_cfg(pvt->F2, reg, &pvt->dcsm0[cs]))
 955                        debugf0("    DCSM0[%d]=0x%08x reg: F2x%x\n",
 956                                cs, pvt->dcsm0[cs], reg);
 957
 958                /* If DCT are NOT ganged, then read in DCT1's mask */
 959                if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) {
 960                        reg = F10_DCSM1 + (cs * 4);
 961                        if (!amd64_read_pci_cfg(pvt->F2, reg,
 962                                                &pvt->dcsm1[cs]))
 963                                debugf0("    DCSM1[%d]=0x%08x reg: F2x%x\n",
 964                                        cs, pvt->dcsm1[cs], reg);
 965                } else {
 966                        pvt->dcsm1[cs] = 0;
 967                }
 968        }
 969}
 970
 971static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
 972{
 973        enum mem_type type;
 974
 975        if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) {
 976                if (pvt->dchr0 & DDR3_MODE)
 977                        type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
 978                else
 979                        type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
 980        } else {
 981                type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
 982        }
 983
 984        amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
 985
 986        return type;
 987}
 988
 989/*
 990 * Read the DRAM Configuration Low register. It differs between CG, D & E revs
 991 * and the later RevF memory controllers (DDR vs DDR2)
 992 *
 993 * Return:
 994 *      number of memory channels in operation
 995 * Pass back:
 996 *      contents of the DCL0_LOW register
 997 */
 998static int k8_early_channel_count(struct amd64_pvt *pvt)
 999{
1000        int flag, err = 0;
1001
1002        err = amd64_read_pci_cfg(pvt->F2, F10_DCLR_0, &pvt->dclr0);
1003        if (err)
1004                return err;
1005
1006        if (pvt->ext_model >= K8_REV_F)
1007                /* RevF (NPT) and later */
1008                flag = pvt->dclr0 & F10_WIDTH_128;
1009        else
1010                /* RevE and earlier */
1011                flag = pvt->dclr0 & REVE_WIDTH_128;
1012
1013        /* not used */
1014        pvt->dclr1 = 0;
1015
1016        return (flag) ? 2 : 1;
1017}
1018
1019/* extract the ERROR ADDRESS for the K8 CPUs */
1020static u64 k8_get_error_address(struct mem_ctl_info *mci,
1021                                struct err_regs *info)
1022{
1023        return (((u64) (info->nbeah & 0xff)) << 32) +
1024                        (info->nbeal & ~0x03);
1025}
1026
1027/*
1028 * Read the Base and Limit registers for K8 based Memory controllers; extract
1029 * fields from the 'raw' reg into separate data fields
1030 *
1031 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN
1032 */
1033static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram)
1034{
1035        u32 low;
1036        u32 off = dram << 3;    /* 8 bytes between DRAM entries */
1037
1038        amd64_read_pci_cfg(pvt->F1, K8_DRAM_BASE_LOW + off, &low);
1039
1040        /* Extract parts into separate data entries */
1041        pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8;
1042        pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7;
1043        pvt->dram_rw_en[dram] = (low & 0x3);
1044
1045        amd64_read_pci_cfg(pvt->F1, K8_DRAM_LIMIT_LOW + off, &low);
1046
1047        /*
1048         * Extract parts into separate data entries. Limit is the HIGHEST memory
1049         * location of the region, so lower 24 bits need to be all ones
1050         */
1051        pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF;
1052        pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7;
1053        pvt->dram_DstNode[dram] = (low & 0x7);
1054}
1055
1056static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
1057                                    struct err_regs *err_info, u64 sys_addr)
1058{
1059        struct mem_ctl_info *src_mci;
1060        int channel, csrow;
1061        u32 page, offset;
1062        u16 syndrome;
1063
1064        syndrome = extract_syndrome(err_info);
1065
1066        /* CHIPKILL enabled */
1067        if (err_info->nbcfg & K8_NBCFG_CHIPKILL) {
1068                channel = get_channel_from_ecc_syndrome(mci, syndrome);
1069                if (channel < 0) {
1070                        /*
1071                         * Syndrome didn't map, so we don't know which of the
1072                         * 2 DIMMs is in error. So we need to ID 'both' of them
1073                         * as suspect.
1074                         */
1075                        amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1076                                           "error reporting race\n", syndrome);
1077                        edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1078                        return;
1079                }
1080        } else {
1081                /*
1082                 * non-chipkill ecc mode
1083                 *
1084                 * The k8 documentation is unclear about how to determine the
1085                 * channel number when using non-chipkill memory.  This method
1086                 * was obtained from email communication with someone at AMD.
1087                 * (Wish the email was placed in this comment - norsk)
1088                 */
1089                channel = ((sys_addr & BIT(3)) != 0);
1090        }
1091
1092        /*
1093         * Find out which node the error address belongs to. This may be
1094         * different from the node that detected the error.
1095         */
1096        src_mci = find_mc_by_sys_addr(mci, sys_addr);
1097        if (!src_mci) {
1098                amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1099                             (unsigned long)sys_addr);
1100                edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1101                return;
1102        }
1103
1104        /* Now map the sys_addr to a CSROW */
1105        csrow = sys_addr_to_csrow(src_mci, sys_addr);
1106        if (csrow < 0) {
1107                edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1108        } else {
1109                error_address_to_page_and_offset(sys_addr, &page, &offset);
1110
1111                edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1112                                  channel, EDAC_MOD_STR);
1113        }
1114}
1115
1116static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
1117{
1118        int *dbam_map;
1119
1120        if (pvt->ext_model >= K8_REV_F)
1121                dbam_map = ddr2_dbam;
1122        else if (pvt->ext_model >= K8_REV_D)
1123                dbam_map = ddr2_dbam_revD;
1124        else
1125                dbam_map = ddr2_dbam_revCG;
1126
1127        return dbam_map[cs_mode];
1128}
1129
1130/*
1131 * Get the number of DCT channels in use.
1132 *
1133 * Return:
1134 *      number of Memory Channels in operation
1135 * Pass back:
1136 *      contents of the DCL0_LOW register
1137 */
1138static int f10_early_channel_count(struct amd64_pvt *pvt)
1139{
1140        int dbams[] = { DBAM0, DBAM1 };
1141        int i, j, channels = 0;
1142        u32 dbam;
1143
1144        /* If we are in 128 bit mode, then we are using 2 channels */
1145        if (pvt->dclr0 & F10_WIDTH_128) {
1146                channels = 2;
1147                return channels;
1148        }
1149
1150        /*
1151         * Need to check if in unganged mode: In such, there are 2 channels,
1152         * but they are not in 128 bit mode and thus the above 'dclr0' status
1153         * bit will be OFF.
1154         *
1155         * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1156         * their CSEnable bit on. If so, then SINGLE DIMM case.
1157         */
1158        debugf0("Data width is not 128 bits - need more decoding\n");
1159
1160        /*
1161         * Check DRAM Bank Address Mapping values for each DIMM to see if there
1162         * is more than just one DIMM present in unganged mode. Need to check
1163         * both controllers since DIMMs can be placed in either one.
1164         */
1165        for (i = 0; i < ARRAY_SIZE(dbams); i++) {
1166                if (amd64_read_pci_cfg(pvt->F2, dbams[i], &dbam))
1167                        goto err_reg;
1168
1169                for (j = 0; j < 4; j++) {
1170                        if (DBAM_DIMM(j, dbam) > 0) {
1171                                channels++;
1172                                break;
1173                        }
1174                }
1175        }
1176
1177        if (channels > 2)
1178                channels = 2;
1179
1180        amd64_info("MCT channel count: %d\n", channels);
1181
1182        return channels;
1183
1184err_reg:
1185        return -1;
1186
1187}
1188
1189static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
1190{
1191        int *dbam_map;
1192
1193        if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1194                dbam_map = ddr3_dbam;
1195        else
1196                dbam_map = ddr2_dbam;
1197
1198        return dbam_map[cs_mode];
1199}
1200
1201static u64 f10_get_error_address(struct mem_ctl_info *mci,
1202                        struct err_regs *info)
1203{
1204        return (((u64) (info->nbeah & 0xffff)) << 32) +
1205                        (info->nbeal & ~0x01);
1206}
1207
1208/*
1209 * Read the Base and Limit registers for F10 based Memory controllers. Extract
1210 * fields from the 'raw' reg into separate data fields.
1211 *
1212 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN.
1213 */
1214static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram)
1215{
1216        u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit;
1217
1218        low_offset = K8_DRAM_BASE_LOW + (dram << 3);
1219        high_offset = F10_DRAM_BASE_HIGH + (dram << 3);
1220
1221        /* read the 'raw' DRAM BASE Address register */
1222        amd64_read_pci_cfg(pvt->F1, low_offset, &low_base);
1223        amd64_read_pci_cfg(pvt->F1, high_offset, &high_base);
1224
1225        /* Extract parts into separate data entries */
1226        pvt->dram_rw_en[dram] = (low_base & 0x3);
1227
1228        if (pvt->dram_rw_en[dram] == 0)
1229                return;
1230
1231        pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7;
1232
1233        pvt->dram_base[dram] = (((u64)high_base & 0x000000FF) << 40) |
1234                               (((u64)low_base  & 0xFFFF0000) << 8);
1235
1236        low_offset = K8_DRAM_LIMIT_LOW + (dram << 3);
1237        high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3);
1238
1239        /* read the 'raw' LIMIT registers */
1240        amd64_read_pci_cfg(pvt->F1, low_offset, &low_limit);
1241        amd64_read_pci_cfg(pvt->F1, high_offset, &high_limit);
1242
1243        pvt->dram_DstNode[dram] = (low_limit & 0x7);
1244        pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7;
1245
1246        /*
1247         * Extract address values and form a LIMIT address. Limit is the HIGHEST
1248         * memory location of the region, so low 24 bits need to be all ones.
1249         */
1250        pvt->dram_limit[dram] = (((u64)high_limit & 0x000000FF) << 40) |
1251                                (((u64) low_limit & 0xFFFF0000) << 8) |
1252                                0x00FFFFFF;
1253}
1254
1255static void f10_read_dram_ctl_register(struct amd64_pvt *pvt)
1256{
1257
1258        if (!amd64_read_pci_cfg(pvt->F2, F10_DCTL_SEL_LOW,
1259                                &pvt->dram_ctl_select_low)) {
1260                debugf0("F2x110 (DCTL Sel. Low): 0x%08x, "
1261                        "High range addresses at: 0x%x\n",
1262                        pvt->dram_ctl_select_low,
1263                        dct_sel_baseaddr(pvt));
1264
1265                debugf0("  DCT mode: %s, All DCTs on: %s\n",
1266                        (dct_ganging_enabled(pvt) ? "ganged" : "unganged"),
1267                        (dct_dram_enabled(pvt) ? "yes"   : "no"));
1268
1269                if (!dct_ganging_enabled(pvt))
1270                        debugf0("  Address range split per DCT: %s\n",
1271                                (dct_high_range_enabled(pvt) ? "yes" : "no"));
1272
1273                debugf0("  DCT data interleave for ECC: %s, "
1274                        "DRAM cleared since last warm reset: %s\n",
1275                        (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1276                        (dct_memory_cleared(pvt) ? "yes" : "no"));
1277
1278                debugf0("  DCT channel interleave: %s, "
1279                        "DCT interleave bits selector: 0x%x\n",
1280                        (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1281                        dct_sel_interleave_addr(pvt));
1282        }
1283
1284        amd64_read_pci_cfg(pvt->F2, F10_DCTL_SEL_HIGH,
1285                           &pvt->dram_ctl_select_high);
1286}
1287
1288/*
1289 * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1290 * Interleaving Modes.
1291 */
1292static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1293                                int hi_range_sel, u32 intlv_en)
1294{
1295        u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1;
1296
1297        if (dct_ganging_enabled(pvt))
1298                cs = 0;
1299        else if (hi_range_sel)
1300                cs = dct_sel_high;
1301        else if (dct_interleave_enabled(pvt)) {
1302                /*
1303                 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1304                 */
1305                if (dct_sel_interleave_addr(pvt) == 0)
1306                        cs = sys_addr >> 6 & 1;
1307                else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) {
1308                        temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1309
1310                        if (dct_sel_interleave_addr(pvt) & 1)
1311                                cs = (sys_addr >> 9 & 1) ^ temp;
1312                        else
1313                                cs = (sys_addr >> 6 & 1) ^ temp;
1314                } else if (intlv_en & 4)
1315                        cs = sys_addr >> 15 & 1;
1316                else if (intlv_en & 2)
1317                        cs = sys_addr >> 14 & 1;
1318                else if (intlv_en & 1)
1319                        cs = sys_addr >> 13 & 1;
1320                else
1321                        cs = sys_addr >> 12 & 1;
1322        } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt))
1323                cs = ~dct_sel_high & 1;
1324        else
1325                cs = 0;
1326
1327        return cs;
1328}
1329
1330static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en)
1331{
1332        if (intlv_en == 1)
1333                return 1;
1334        else if (intlv_en == 3)
1335                return 2;
1336        else if (intlv_en == 7)
1337                return 3;
1338
1339        return 0;
1340}
1341
1342/* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */
1343static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel,
1344                                                 u32 dct_sel_base_addr,
1345                                                 u64 dct_sel_base_off,
1346                                                 u32 hole_valid, u32 hole_off,
1347                                                 u64 dram_base)
1348{
1349        u64 chan_off;
1350
1351        if (hi_range_sel) {
1352                if (!(dct_sel_base_addr & 0xFFFF0000) &&
1353                   hole_valid && (sys_addr >= 0x100000000ULL))
1354                        chan_off = hole_off << 16;
1355                else
1356                        chan_off = dct_sel_base_off;
1357        } else {
1358                if (hole_valid && (sys_addr >= 0x100000000ULL))
1359                        chan_off = hole_off << 16;
1360                else
1361                        chan_off = dram_base & 0xFFFFF8000000ULL;
1362        }
1363
1364        return (sys_addr & 0x0000FFFFFFFFFFC0ULL) -
1365                        (chan_off & 0x0000FFFFFF800000ULL);
1366}
1367
1368/* Hack for the time being - Can we get this from BIOS?? */
1369#define CH0SPARE_RANK   0
1370#define CH1SPARE_RANK   1
1371
1372/*
1373 * checks if the csrow passed in is marked as SPARED, if so returns the new
1374 * spare row
1375 */
1376static inline int f10_process_possible_spare(int csrow,
1377                                u32 cs, struct amd64_pvt *pvt)
1378{
1379        u32 swap_done;
1380        u32 bad_dram_cs;
1381
1382        /* Depending on channel, isolate respective SPARING info */
1383        if (cs) {
1384                swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare);
1385                bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare);
1386                if (swap_done && (csrow == bad_dram_cs))
1387                        csrow = CH1SPARE_RANK;
1388        } else {
1389                swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare);
1390                bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare);
1391                if (swap_done && (csrow == bad_dram_cs))
1392                        csrow = CH0SPARE_RANK;
1393        }
1394        return csrow;
1395}
1396
1397/*
1398 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1399 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1400 *
1401 * Return:
1402 *      -EINVAL:  NOT FOUND
1403 *      0..csrow = Chip-Select Row
1404 */
1405static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs)
1406{
1407        struct mem_ctl_info *mci;
1408        struct amd64_pvt *pvt;
1409        u32 cs_base, cs_mask;
1410        int cs_found = -EINVAL;
1411        int csrow;
1412
1413        mci = mcis[nid];
1414        if (!mci)
1415                return cs_found;
1416
1417        pvt = mci->pvt_info;
1418
1419        debugf1("InputAddr=0x%x  channelselect=%d\n", in_addr, cs);
1420
1421        for (csrow = 0; csrow < pvt->cs_count; csrow++) {
1422
1423                cs_base = amd64_get_dct_base(pvt, cs, csrow);
1424                if (!(cs_base & K8_DCSB_CS_ENABLE))
1425                        continue;
1426
1427                /*
1428                 * We have an ENABLED CSROW, Isolate just the MASK bits of the
1429                 * target: [28:19] and [13:5], which map to [36:27] and [21:13]
1430                 * of the actual address.
1431                 */
1432                cs_base &= REV_F_F1Xh_DCSB_BASE_BITS;
1433
1434                /*
1435                 * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and
1436                 * [4:0] to become ON. Then mask off bits [28:0] ([36:8])
1437                 */
1438                cs_mask = amd64_get_dct_mask(pvt, cs, csrow);
1439
1440                debugf1("    CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n",
1441                                csrow, cs_base, cs_mask);
1442
1443                cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF;
1444
1445                debugf1("              Final CSMask=0x%x\n", cs_mask);
1446                debugf1("    (InputAddr & ~CSMask)=0x%x "
1447                                "(CSBase & ~CSMask)=0x%x\n",
1448                                (in_addr & ~cs_mask), (cs_base & ~cs_mask));
1449
1450                if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) {
1451                        cs_found = f10_process_possible_spare(csrow, cs, pvt);
1452
1453                        debugf1(" MATCH csrow=%d\n", cs_found);
1454                        break;
1455                }
1456        }
1457        return cs_found;
1458}
1459
1460/* For a given @dram_range, check if @sys_addr falls within it. */
1461static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range,
1462                                  u64 sys_addr, int *nid, int *chan_sel)
1463{
1464        int node_id, cs_found = -EINVAL, high_range = 0;
1465        u32 intlv_en, intlv_sel, intlv_shift, hole_off;
1466        u32 hole_valid, tmp, dct_sel_base, channel;
1467        u64 dram_base, chan_addr, dct_sel_base_off;
1468
1469        dram_base = pvt->dram_base[dram_range];
1470        intlv_en = pvt->dram_IntlvEn[dram_range];
1471
1472        node_id = pvt->dram_DstNode[dram_range];
1473        intlv_sel = pvt->dram_IntlvSel[dram_range];
1474
1475        debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n",
1476                dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]);
1477
1478        /*
1479         * This assumes that one node's DHAR is the same as all the other
1480         * nodes' DHAR.
1481         */
1482        hole_off = (pvt->dhar & 0x0000FF80);
1483        hole_valid = (pvt->dhar & 0x1);
1484        dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16;
1485
1486        debugf1("   HoleOffset=0x%x  HoleValid=0x%x IntlvSel=0x%x\n",
1487                        hole_off, hole_valid, intlv_sel);
1488
1489        if (intlv_en &&
1490            (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1491                return -EINVAL;
1492
1493        dct_sel_base = dct_sel_baseaddr(pvt);
1494
1495        /*
1496         * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1497         * select between DCT0 and DCT1.
1498         */
1499        if (dct_high_range_enabled(pvt) &&
1500           !dct_ganging_enabled(pvt) &&
1501           ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1502                high_range = 1;
1503
1504        channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en);
1505
1506        chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base,
1507                                             dct_sel_base_off, hole_valid,
1508                                             hole_off, dram_base);
1509
1510        intlv_shift = f10_map_intlv_en_to_shift(intlv_en);
1511
1512        /* remove Node ID (in case of memory interleaving) */
1513        tmp = chan_addr & 0xFC0;
1514
1515        chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp;
1516
1517        /* remove channel interleave and hash */
1518        if (dct_interleave_enabled(pvt) &&
1519           !dct_high_range_enabled(pvt) &&
1520           !dct_ganging_enabled(pvt)) {
1521                if (dct_sel_interleave_addr(pvt) != 1)
1522                        chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL;
1523                else {
1524                        tmp = chan_addr & 0xFC0;
1525                        chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1)
1526                                        | tmp;
1527                }
1528        }
1529
1530        debugf1("   (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n",
1531                chan_addr, (u32)(chan_addr >> 8));
1532
1533        cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel);
1534
1535        if (cs_found >= 0) {
1536                *nid = node_id;
1537                *chan_sel = channel;
1538        }
1539        return cs_found;
1540}
1541
1542static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1543                                       int *node, int *chan_sel)
1544{
1545        int dram_range, cs_found = -EINVAL;
1546        u64 dram_base, dram_limit;
1547
1548        for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) {
1549
1550                if (!pvt->dram_rw_en[dram_range])
1551                        continue;
1552
1553                dram_base = pvt->dram_base[dram_range];
1554                dram_limit = pvt->dram_limit[dram_range];
1555
1556                if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) {
1557
1558                        cs_found = f10_match_to_this_node(pvt, dram_range,
1559                                                          sys_addr, node,
1560                                                          chan_sel);
1561                        if (cs_found >= 0)
1562                                break;
1563                }
1564        }
1565        return cs_found;
1566}
1567
1568/*
1569 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1570 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1571 *
1572 * The @sys_addr is usually an error address received from the hardware
1573 * (MCX_ADDR).
1574 */
1575static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
1576                                     struct err_regs *err_info,
1577                                     u64 sys_addr)
1578{
1579        struct amd64_pvt *pvt = mci->pvt_info;
1580        u32 page, offset;
1581        int nid, csrow, chan = 0;
1582        u16 syndrome;
1583
1584        csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1585
1586        if (csrow < 0) {
1587                edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1588                return;
1589        }
1590
1591        error_address_to_page_and_offset(sys_addr, &page, &offset);
1592
1593        syndrome = extract_syndrome(err_info);
1594
1595        /*
1596         * We need the syndromes for channel detection only when we're
1597         * ganged. Otherwise @chan should already contain the channel at
1598         * this point.
1599         */
1600        if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL))
1601                chan = get_channel_from_ecc_syndrome(mci, syndrome);
1602
1603        if (chan >= 0)
1604                edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1605                                  EDAC_MOD_STR);
1606        else
1607                /*
1608                 * Channel unknown, report all channels on this CSROW as failed.
1609                 */
1610                for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1611                        edac_mc_handle_ce(mci, page, offset, syndrome,
1612                                          csrow, chan, EDAC_MOD_STR);
1613}
1614
1615/*
1616 * debug routine to display the memory sizes of all logical DIMMs and its
1617 * CSROWs as well
1618 */
1619static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt)
1620{
1621        int dimm, size0, size1, factor = 0;
1622        u32 dbam;
1623        u32 *dcsb;
1624
1625        if (boot_cpu_data.x86 == 0xf) {
1626                if (pvt->dclr0 & F10_WIDTH_128)
1627                        factor = 1;
1628
1629                /* K8 families < revF not supported yet */
1630               if (pvt->ext_model < K8_REV_F)
1631                        return;
1632               else
1633                       WARN_ON(ctrl != 0);
1634        }
1635
1636        dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1637        dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dcsb1 : pvt->dcsb0;
1638
1639        debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
1640
1641        edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1642
1643        /* Dump memory sizes for DIMM and its CSROWs */
1644        for (dimm = 0; dimm < 4; dimm++) {
1645
1646                size0 = 0;
1647                if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE)
1648                        size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
1649
1650                size1 = 0;
1651                if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE)
1652                        size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
1653
1654                amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1655                                dimm * 2,     size0 << factor,
1656                                dimm * 2 + 1, size1 << factor);
1657        }
1658}
1659
1660static struct amd64_family_type amd64_family_types[] = {
1661        [K8_CPUS] = {
1662                .ctl_name = "K8",
1663                .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1664                .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1665                .ops = {
1666                        .early_channel_count    = k8_early_channel_count,
1667                        .get_error_address      = k8_get_error_address,
1668                        .read_dram_base_limit   = k8_read_dram_base_limit,
1669                        .map_sysaddr_to_csrow   = k8_map_sysaddr_to_csrow,
1670                        .dbam_to_cs             = k8_dbam_to_chip_select,
1671                }
1672        },
1673        [F10_CPUS] = {
1674                .ctl_name = "F10h",
1675                .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1676                .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1677                .ops = {
1678                        .early_channel_count    = f10_early_channel_count,
1679                        .get_error_address      = f10_get_error_address,
1680                        .read_dram_base_limit   = f10_read_dram_base_limit,
1681                        .read_dram_ctl_register = f10_read_dram_ctl_register,
1682                        .map_sysaddr_to_csrow   = f10_map_sysaddr_to_csrow,
1683                        .dbam_to_cs             = f10_dbam_to_chip_select,
1684                }
1685        },
1686};
1687
1688static struct pci_dev *pci_get_related_function(unsigned int vendor,
1689                                                unsigned int device,
1690                                                struct pci_dev *related)
1691{
1692        struct pci_dev *dev = NULL;
1693
1694        dev = pci_get_device(vendor, device, dev);
1695        while (dev) {
1696                if ((dev->bus->number == related->bus->number) &&
1697                    (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1698                        break;
1699                dev = pci_get_device(vendor, device, dev);
1700        }
1701
1702        return dev;
1703}
1704
1705/*
1706 * These are tables of eigenvectors (one per line) which can be used for the
1707 * construction of the syndrome tables. The modified syndrome search algorithm
1708 * uses those to find the symbol in error and thus the DIMM.
1709 *
1710 * Algorithm courtesy of Ross LaFetra from AMD.
1711 */
1712static u16 x4_vectors[] = {
1713        0x2f57, 0x1afe, 0x66cc, 0xdd88,
1714        0x11eb, 0x3396, 0x7f4c, 0xeac8,
1715        0x0001, 0x0002, 0x0004, 0x0008,
1716        0x1013, 0x3032, 0x4044, 0x8088,
1717        0x106b, 0x30d6, 0x70fc, 0xe0a8,
1718        0x4857, 0xc4fe, 0x13cc, 0x3288,
1719        0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1720        0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1721        0x15c1, 0x2a42, 0x89ac, 0x4758,
1722        0x2b03, 0x1602, 0x4f0c, 0xca08,
1723        0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1724        0x8ba7, 0x465e, 0x244c, 0x1cc8,
1725        0x2b87, 0x164e, 0x642c, 0xdc18,
1726        0x40b9, 0x80de, 0x1094, 0x20e8,
1727        0x27db, 0x1eb6, 0x9dac, 0x7b58,
1728        0x11c1, 0x2242, 0x84ac, 0x4c58,
1729        0x1be5, 0x2d7a, 0x5e34, 0xa718,
1730        0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1731        0x4c97, 0xc87e, 0x11fc, 0x33a8,
1732        0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1733        0x16b3, 0x3d62, 0x4f34, 0x8518,
1734        0x1e2f, 0x391a, 0x5cac, 0xf858,
1735        0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1736        0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1737        0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1738        0x4397, 0xc27e, 0x17fc, 0x3ea8,
1739        0x1617, 0x3d3e, 0x6464, 0xb8b8,
1740        0x23ff, 0x12aa, 0xab6c, 0x56d8,
1741        0x2dfb, 0x1ba6, 0x913c, 0x7328,
1742        0x185d, 0x2ca6, 0x7914, 0x9e28,
1743        0x171b, 0x3e36, 0x7d7c, 0xebe8,
1744        0x4199, 0x82ee, 0x19f4, 0x2e58,
1745        0x4807, 0xc40e, 0x130c, 0x3208,
1746        0x1905, 0x2e0a, 0x5804, 0xac08,
1747        0x213f, 0x132a, 0xadfc, 0x5ba8,
1748        0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1749};
1750
1751static u16 x8_vectors[] = {
1752        0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1753        0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1754        0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1755        0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1756        0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1757        0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1758        0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1759        0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1760        0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1761        0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1762        0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1763        0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1764        0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1765        0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1766        0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1767        0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1768        0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1769        0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1770        0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1771};
1772
1773static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs,
1774                           int v_dim)
1775{
1776        unsigned int i, err_sym;
1777
1778        for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1779                u16 s = syndrome;
1780                int v_idx =  err_sym * v_dim;
1781                int v_end = (err_sym + 1) * v_dim;
1782
1783                /* walk over all 16 bits of the syndrome */
1784                for (i = 1; i < (1U << 16); i <<= 1) {
1785
1786                        /* if bit is set in that eigenvector... */
1787                        if (v_idx < v_end && vectors[v_idx] & i) {
1788                                u16 ev_comp = vectors[v_idx++];
1789
1790                                /* ... and bit set in the modified syndrome, */
1791                                if (s & i) {
1792                                        /* remove it. */
1793                                        s ^= ev_comp;
1794
1795                                        if (!s)
1796                                                return err_sym;
1797                                }
1798
1799                        } else if (s & i)
1800                                /* can't get to zero, move to next symbol */
1801                                break;
1802                }
1803        }
1804
1805        debugf0("syndrome(%x) not found\n", syndrome);
1806        return -1;
1807}
1808
1809static int map_err_sym_to_channel(int err_sym, int sym_size)
1810{
1811        if (sym_size == 4)
1812                switch (err_sym) {
1813                case 0x20:
1814                case 0x21:
1815                        return 0;
1816                        break;
1817                case 0x22:
1818                case 0x23:
1819                        return 1;
1820                        break;
1821                default:
1822                        return err_sym >> 4;
1823                        break;
1824                }
1825        /* x8 symbols */
1826        else
1827                switch (err_sym) {
1828                /* imaginary bits not in a DIMM */
1829                case 0x10:
1830                        WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1831                                          err_sym);
1832                        return -1;
1833                        break;
1834
1835                case 0x11:
1836                        return 0;
1837                        break;
1838                case 0x12:
1839                        return 1;
1840                        break;
1841                default:
1842                        return err_sym >> 3;
1843                        break;
1844                }
1845        return -1;
1846}
1847
1848static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1849{
1850        struct amd64_pvt *pvt = mci->pvt_info;
1851        int err_sym = -1;
1852
1853        if (pvt->syn_type == 8)
1854                err_sym = decode_syndrome(syndrome, x8_vectors,
1855                                          ARRAY_SIZE(x8_vectors),
1856                                          pvt->syn_type);
1857        else if (pvt->syn_type == 4)
1858                err_sym = decode_syndrome(syndrome, x4_vectors,
1859                                          ARRAY_SIZE(x4_vectors),
1860                                          pvt->syn_type);
1861        else {
1862                amd64_warn("Illegal syndrome type: %u\n", pvt->syn_type);
1863                return err_sym;
1864        }
1865
1866        return map_err_sym_to_channel(err_sym, pvt->syn_type);
1867}
1868
1869/*
1870 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1871 * ADDRESS and process.
1872 */
1873static void amd64_handle_ce(struct mem_ctl_info *mci,
1874                            struct err_regs *info)
1875{
1876        struct amd64_pvt *pvt = mci->pvt_info;
1877        u64 sys_addr;
1878
1879        /* Ensure that the Error Address is VALID */
1880        if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) {
1881                amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1882                edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1883                return;
1884        }
1885
1886        sys_addr = pvt->ops->get_error_address(mci, info);
1887
1888        amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
1889
1890        pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr);
1891}
1892
1893/* Handle any Un-correctable Errors (UEs) */
1894static void amd64_handle_ue(struct mem_ctl_info *mci,
1895                            struct err_regs *info)
1896{
1897        struct amd64_pvt *pvt = mci->pvt_info;
1898        struct mem_ctl_info *log_mci, *src_mci = NULL;
1899        int csrow;
1900        u64 sys_addr;
1901        u32 page, offset;
1902
1903        log_mci = mci;
1904
1905        if (!(info->nbsh & K8_NBSH_VALID_ERROR_ADDR)) {
1906                amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1907                edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1908                return;
1909        }
1910
1911        sys_addr = pvt->ops->get_error_address(mci, info);
1912
1913        /*
1914         * Find out which node the error address belongs to. This may be
1915         * different from the node that detected the error.
1916         */
1917        src_mci = find_mc_by_sys_addr(mci, sys_addr);
1918        if (!src_mci) {
1919                amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1920                                  (unsigned long)sys_addr);
1921                edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1922                return;
1923        }
1924
1925        log_mci = src_mci;
1926
1927        csrow = sys_addr_to_csrow(log_mci, sys_addr);
1928        if (csrow < 0) {
1929                amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1930                                  (unsigned long)sys_addr);
1931                edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1932        } else {
1933                error_address_to_page_and_offset(sys_addr, &page, &offset);
1934                edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1935        }
1936}
1937
1938static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1939                                            struct err_regs *info)
1940{
1941        u16 ec = EC(info->nbsl);
1942        u8 xec = XEC(info->nbsl, 0x1f);
1943        int ecc_type = (info->nbsh >> 13) & 0x3;
1944
1945        /* Bail early out if this was an 'observed' error */
1946        if (PP(ec) == K8_NBSL_PP_OBS)
1947                return;
1948
1949        /* Do only ECC errors */
1950        if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1951                return;
1952
1953        if (ecc_type == 2)
1954                amd64_handle_ce(mci, info);
1955        else if (ecc_type == 1)
1956                amd64_handle_ue(mci, info);
1957}
1958
1959void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
1960{
1961        struct mem_ctl_info *mci = mcis[node_id];
1962        struct err_regs regs;
1963
1964        regs.nbsl  = (u32) m->status;
1965        regs.nbsh  = (u32)(m->status >> 32);
1966        regs.nbeal = (u32) m->addr;
1967        regs.nbeah = (u32)(m->addr >> 32);
1968        regs.nbcfg = nbcfg;
1969
1970        __amd64_decode_bus_error(mci, &regs);
1971
1972        /*
1973         * Check the UE bit of the NB status high register, if set generate some
1974         * logs. If NOT a GART error, then process the event as a NO-INFO event.
1975         * If it was a GART error, skip that process.
1976         *
1977         * FIXME: this should go somewhere else, if at all.
1978         */
1979        if (regs.nbsh & K8_NBSH_UC_ERR && !report_gart_errors)
1980                edac_mc_handle_ue_no_info(mci, "UE bit is set");
1981
1982}
1983
1984/*
1985 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1986 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1987 */
1988static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1989{
1990        /* Reserve the ADDRESS MAP Device */
1991        pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1992        if (!pvt->F1) {
1993                amd64_err("error address map device not found: "
1994                          "vendor %x device 0x%x (broken BIOS?)\n",
1995                          PCI_VENDOR_ID_AMD, f1_id);
1996                return -ENODEV;
1997        }
1998
1999        /* Reserve the MISC Device */
2000        pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
2001        if (!pvt->F3) {
2002                pci_dev_put(pvt->F1);
2003                pvt->F1 = NULL;
2004
2005                amd64_err("error F3 device not found: "
2006                          "vendor %x device 0x%x (broken BIOS?)\n",
2007                          PCI_VENDOR_ID_AMD, f3_id);
2008
2009                return -ENODEV;
2010        }
2011        debugf1("F1: %s\n", pci_name(pvt->F1));
2012        debugf1("F2: %s\n", pci_name(pvt->F2));
2013        debugf1("F3: %s\n", pci_name(pvt->F3));
2014
2015        return 0;
2016}
2017
2018static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2019{
2020        pci_dev_put(pvt->F1);
2021        pci_dev_put(pvt->F3);
2022}
2023
2024/*
2025 * Retrieve the hardware registers of the memory controller (this includes the
2026 * 'Address Map' and 'Misc' device regs)
2027 */
2028static void read_mc_regs(struct amd64_pvt *pvt)
2029{
2030        u64 msr_val;
2031        u32 tmp;
2032        int dram;
2033
2034        /*
2035         * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2036         * those are Read-As-Zero
2037         */
2038        rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2039        debugf0("  TOP_MEM:  0x%016llx\n", pvt->top_mem);
2040
2041        /* check first whether TOP_MEM2 is enabled */
2042        rdmsrl(MSR_K8_SYSCFG, msr_val);
2043        if (msr_val & (1U << 21)) {
2044                rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2045                debugf0("  TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2046        } else
2047                debugf0("  TOP_MEM2 disabled.\n");
2048
2049        amd64_read_pci_cfg(pvt->F3, K8_NBCAP, &pvt->nbcap);
2050
2051        if (pvt->ops->read_dram_ctl_register)
2052                pvt->ops->read_dram_ctl_register(pvt);
2053
2054        for (dram = 0; dram < DRAM_REG_COUNT; dram++) {
2055                /*
2056                 * Call CPU specific READ function to get the DRAM Base and
2057                 * Limit values from the DCT.
2058                 */
2059                pvt->ops->read_dram_base_limit(pvt, dram);
2060
2061                /*
2062                 * Only print out debug info on rows with both R and W Enabled.
2063                 * Normal processing, compiler should optimize this whole 'if'
2064                 * debug output block away.
2065                 */
2066                if (pvt->dram_rw_en[dram] != 0) {
2067                        debugf1("  DRAM-BASE[%d]: 0x%016llx "
2068                                "DRAM-LIMIT:  0x%016llx\n",
2069                                dram,
2070                                pvt->dram_base[dram],
2071                                pvt->dram_limit[dram]);
2072
2073                        debugf1("        IntlvEn=%s %s %s "
2074                                "IntlvSel=%d DstNode=%d\n",
2075                                pvt->dram_IntlvEn[dram] ?
2076                                        "Enabled" : "Disabled",
2077                                (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W",
2078                                (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R",
2079                                pvt->dram_IntlvSel[dram],
2080                                pvt->dram_DstNode[dram]);
2081                }
2082        }
2083
2084        amd64_read_dct_base_mask(pvt);
2085
2086        amd64_read_pci_cfg(pvt->F1, K8_DHAR, &pvt->dhar);
2087        amd64_read_dbam_reg(pvt);
2088
2089        amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2090
2091        amd64_read_pci_cfg(pvt->F2, F10_DCLR_0, &pvt->dclr0);
2092        amd64_read_pci_cfg(pvt->F2, F10_DCHR_0, &pvt->dchr0);
2093
2094        if (boot_cpu_data.x86 >= 0x10) {
2095                if (!dct_ganging_enabled(pvt)) {
2096                        amd64_read_pci_cfg(pvt->F2, F10_DCLR_1, &pvt->dclr1);
2097                        amd64_read_pci_cfg(pvt->F2, F10_DCHR_1, &pvt->dchr1);
2098                }
2099                amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2100        }
2101
2102        if (boot_cpu_data.x86 == 0x10 &&
2103            boot_cpu_data.x86_model > 7 &&
2104            /* F3x180[EccSymbolSize]=1 => x8 symbols */
2105            tmp & BIT(25))
2106                pvt->syn_type = 8;
2107        else
2108                pvt->syn_type = 4;
2109
2110        amd64_dump_misc_regs(pvt);
2111}
2112
2113/*
2114 * NOTE: CPU Revision Dependent code
2115 *
2116 * Input:
2117 *      @csrow_nr ChipSelect Row Number (0..pvt->cs_count-1)
2118 *      k8 private pointer to -->
2119 *                      DRAM Bank Address mapping register
2120 *                      node_id
2121 *                      DCL register where dual_channel_active is
2122 *
2123 * The DBAM register consists of 4 sets of 4 bits each definitions:
2124 *
2125 * Bits:        CSROWs
2126 * 0-3          CSROWs 0 and 1
2127 * 4-7          CSROWs 2 and 3
2128 * 8-11         CSROWs 4 and 5
2129 * 12-15        CSROWs 6 and 7
2130 *
2131 * Values range from: 0 to 15
2132 * The meaning of the values depends on CPU revision and dual-channel state,
2133 * see relevant BKDG more info.
2134 *
2135 * The memory controller provides for total of only 8 CSROWs in its current
2136 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2137 * single channel or two (2) DIMMs in dual channel mode.
2138 *
2139 * The following code logic collapses the various tables for CSROW based on CPU
2140 * revision.
2141 *
2142 * Returns:
2143 *      The number of PAGE_SIZE pages on the specified CSROW number it
2144 *      encompasses
2145 *
2146 */
2147static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt)
2148{
2149        u32 cs_mode, nr_pages;
2150
2151        /*
2152         * The math on this doesn't look right on the surface because x/2*4 can
2153         * be simplified to x*2 but this expression makes use of the fact that
2154         * it is integral math where 1/2=0. This intermediate value becomes the
2155         * number of bits to shift the DBAM register to extract the proper CSROW
2156         * field.
2157         */
2158        cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
2159
2160        nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT);
2161
2162        /*
2163         * If dual channel then double the memory size of single channel.
2164         * Channel count is 1 or 2
2165         */
2166        nr_pages <<= (pvt->channel_count - 1);
2167
2168        debugf0("  (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
2169        debugf0("    nr_pages= %u  channel-count = %d\n",
2170                nr_pages, pvt->channel_count);
2171
2172        return nr_pages;
2173}
2174
2175/*
2176 * Initialize the array of csrow attribute instances, based on the values
2177 * from pci config hardware registers.
2178 */
2179static int init_csrows(struct mem_ctl_info *mci)
2180{
2181        struct csrow_info *csrow;
2182        struct amd64_pvt *pvt = mci->pvt_info;
2183        u64 input_addr_min, input_addr_max, sys_addr;
2184        u32 val;
2185        int i, empty = 1;
2186
2187        amd64_read_pci_cfg(pvt->F3, K8_NBCFG, &val);
2188
2189        pvt->nbcfg = val;
2190        pvt->ctl_error_info.nbcfg = val;
2191
2192        debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2193                pvt->mc_node_id, val,
2194                !!(val & K8_NBCFG_CHIPKILL), !!(val & K8_NBCFG_ECC_ENABLE));
2195
2196        for (i = 0; i < pvt->cs_count; i++) {
2197                csrow = &mci->csrows[i];
2198
2199                if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) {
2200                        debugf1("----CSROW %d EMPTY for node %d\n", i,
2201                                pvt->mc_node_id);
2202                        continue;
2203                }
2204
2205                debugf1("----CSROW %d VALID for MC node %d\n",
2206                        i, pvt->mc_node_id);
2207
2208                empty = 0;
2209                csrow->nr_pages = amd64_csrow_nr_pages(i, pvt);
2210                find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2211                sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2212                csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2213                sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2214                csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2215                csrow->page_mask = ~mask_from_dct_mask(pvt, i);
2216                /* 8 bytes of resolution */
2217
2218                csrow->mtype = amd64_determine_memory_type(pvt, i);
2219
2220                debugf1("  for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2221                debugf1("    input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2222                        (unsigned long)input_addr_min,
2223                        (unsigned long)input_addr_max);
2224                debugf1("    sys_addr: 0x%lx  page_mask: 0x%lx\n",
2225                        (unsigned long)sys_addr, csrow->page_mask);
2226                debugf1("    nr_pages: %u  first_page: 0x%lx "
2227                        "last_page: 0x%lx\n",
2228                        (unsigned)csrow->nr_pages,
2229                        csrow->first_page, csrow->last_page);
2230
2231                /*
2232                 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2233                 */
2234                if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE)
2235                        csrow->edac_mode =
2236                            (pvt->nbcfg & K8_NBCFG_CHIPKILL) ?
2237                            EDAC_S4ECD4ED : EDAC_SECDED;
2238                else
2239                        csrow->edac_mode = EDAC_NONE;
2240        }
2241
2242        return empty;
2243}
2244
2245/* get all cores on this DCT */
2246static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid)
2247{
2248        int cpu;
2249
2250        for_each_online_cpu(cpu)
2251                if (amd_get_nb_id(cpu) == nid)
2252                        cpumask_set_cpu(cpu, mask);
2253}
2254
2255/* check MCG_CTL on all the cpus on this node */
2256static bool amd64_nb_mce_bank_enabled_on_node(int nid)
2257{
2258        cpumask_var_t mask;
2259        int cpu, nbe;
2260        bool ret = false;
2261
2262        if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2263                amd64_warn("%s: Error allocating mask\n", __func__);
2264                return false;
2265        }
2266
2267        get_cpus_on_this_dct_cpumask(mask, nid);
2268
2269        rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2270
2271        for_each_cpu(cpu, mask) {
2272                struct msr *reg = per_cpu_ptr(msrs, cpu);
2273                nbe = reg->l & K8_MSR_MCGCTL_NBE;
2274
2275                debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2276                        cpu, reg->q,
2277                        (nbe ? "enabled" : "disabled"));
2278
2279                if (!nbe)
2280                        goto out;
2281        }
2282        ret = true;
2283
2284out:
2285        free_cpumask_var(mask);
2286        return ret;
2287}
2288
2289static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
2290{
2291        cpumask_var_t cmask;
2292        int cpu;
2293
2294        if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2295                amd64_warn("%s: error allocating mask\n", __func__);
2296                return false;
2297        }
2298
2299        get_cpus_on_this_dct_cpumask(cmask, nid);
2300
2301        rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2302
2303        for_each_cpu(cpu, cmask) {
2304
2305                struct msr *reg = per_cpu_ptr(msrs, cpu);
2306
2307                if (on) {
2308                        if (reg->l & K8_MSR_MCGCTL_NBE)
2309                                s->flags.nb_mce_enable = 1;
2310
2311                        reg->l |= K8_MSR_MCGCTL_NBE;
2312                } else {
2313                        /*
2314                         * Turn off NB MCE reporting only when it was off before
2315                         */
2316                        if (!s->flags.nb_mce_enable)
2317                                reg->l &= ~K8_MSR_MCGCTL_NBE;
2318                }
2319        }
2320        wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2321
2322        free_cpumask_var(cmask);
2323
2324        return 0;
2325}
2326
2327static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2328                                       struct pci_dev *F3)
2329{
2330        bool ret = true;
2331        u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
2332
2333        if (toggle_ecc_err_reporting(s, nid, ON)) {
2334                amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2335                return false;
2336        }
2337
2338        amd64_read_pci_cfg(F3, K8_NBCTL, &value);
2339
2340        /* turn on UECCEn and CECCEn bits */
2341        s->old_nbctl   = value & mask;
2342        s->nbctl_valid = true;
2343
2344        value |= mask;
2345        pci_write_config_dword(F3, K8_NBCTL, value);
2346
2347        amd64_read_pci_cfg(F3, K8_NBCFG, &value);
2348
2349        debugf0("1: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2350                nid, value,
2351                !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE));
2352
2353        if (!(value & K8_NBCFG_ECC_ENABLE)) {
2354                amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2355
2356                s->flags.nb_ecc_prev = 0;
2357
2358                /* Attempt to turn on DRAM ECC Enable */
2359                value |= K8_NBCFG_ECC_ENABLE;
2360                pci_write_config_dword(F3, K8_NBCFG, value);
2361
2362                amd64_read_pci_cfg(F3, K8_NBCFG, &value);
2363
2364                if (!(value & K8_NBCFG_ECC_ENABLE)) {
2365                        amd64_warn("Hardware rejected DRAM ECC enable,"
2366                                   "check memory DIMM configuration.\n");
2367                        ret = false;
2368                } else {
2369                        amd64_info("Hardware accepted DRAM ECC Enable\n");
2370                }
2371        } else {
2372                s->flags.nb_ecc_prev = 1;
2373        }
2374
2375        debugf0("2: node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2376                nid, value,
2377                !!(value & K8_NBCFG_CHIPKILL), !!(value & K8_NBCFG_ECC_ENABLE));
2378
2379        return ret;
2380}
2381
2382static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2383                                        struct pci_dev *F3)
2384{
2385        u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
2386
2387        if (!s->nbctl_valid)
2388                return;
2389
2390        amd64_read_pci_cfg(F3, K8_NBCTL, &value);
2391        value &= ~mask;
2392        value |= s->old_nbctl;
2393
2394        pci_write_config_dword(F3, K8_NBCTL, value);
2395
2396        /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2397        if (!s->flags.nb_ecc_prev) {
2398                amd64_read_pci_cfg(F3, K8_NBCFG, &value);
2399                value &= ~K8_NBCFG_ECC_ENABLE;
2400                pci_write_config_dword(F3, K8_NBCFG, value);
2401        }
2402
2403        /* restore the NB Enable MCGCTL bit */
2404        if (toggle_ecc_err_reporting(s, nid, OFF))
2405                amd64_warn("Error restoring NB MCGCTL settings!\n");
2406}
2407
2408/*
2409 * EDAC requires that the BIOS have ECC enabled before
2410 * taking over the processing of ECC errors. A command line
2411 * option allows to force-enable hardware ECC later in
2412 * enable_ecc_error_reporting().
2413 */
2414static const char *ecc_msg =
2415        "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2416        " Either enable ECC checking or force module loading by setting "
2417        "'ecc_enable_override'.\n"
2418        " (Note that use of the override may cause unknown side effects.)\n";
2419
2420static bool ecc_enabled(struct pci_dev *F3, u8 nid)
2421{
2422        u32 value;
2423        u8 ecc_en = 0;
2424        bool nb_mce_en = false;
2425
2426        amd64_read_pci_cfg(F3, K8_NBCFG, &value);
2427
2428        ecc_en = !!(value & K8_NBCFG_ECC_ENABLE);
2429        amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2430
2431        nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2432        if (!nb_mce_en)
2433                amd64_notice("NB MCE bank disabled, set MSR "
2434                             "0x%08x[4] on node %d to enable.\n",
2435                             MSR_IA32_MCG_CTL, nid);
2436
2437        if (!ecc_en || !nb_mce_en) {
2438                amd64_notice("%s", ecc_msg);
2439                return false;
2440        }
2441        return true;
2442}
2443
2444struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2445                                          ARRAY_SIZE(amd64_inj_attrs) +
2446                                          1];
2447
2448struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2449
2450static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2451{
2452        unsigned int i = 0, j = 0;
2453
2454        for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2455                sysfs_attrs[i] = amd64_dbg_attrs[i];
2456
2457        if (boot_cpu_data.x86 >= 0x10)
2458                for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2459                        sysfs_attrs[i] = amd64_inj_attrs[j];
2460
2461        sysfs_attrs[i] = terminator;
2462
2463        mci->mc_driver_sysfs_attributes = sysfs_attrs;
2464}
2465
2466static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
2467{
2468        struct amd64_pvt *pvt = mci->pvt_info;
2469
2470        mci->mtype_cap          = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2471        mci->edac_ctl_cap       = EDAC_FLAG_NONE;
2472
2473        if (pvt->nbcap & K8_NBCAP_SECDED)
2474                mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2475
2476        if (pvt->nbcap & K8_NBCAP_CHIPKILL)
2477                mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2478
2479        mci->edac_cap           = amd64_determine_edac_cap(pvt);
2480        mci->mod_name           = EDAC_MOD_STR;
2481        mci->mod_ver            = EDAC_AMD64_VERSION;
2482        mci->ctl_name           = pvt->ctl_name;
2483        mci->dev_name           = pci_name(pvt->F2);
2484        mci->ctl_page_to_phys   = NULL;
2485
2486        /* memory scrubber interface */
2487        mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2488        mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2489}
2490
2491/*
2492 * returns a pointer to the family descriptor on success, NULL otherwise.
2493 */
2494static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2495{
2496        u8 fam = boot_cpu_data.x86;
2497        struct amd64_family_type *fam_type = NULL;
2498
2499        switch (fam) {
2500        case 0xf:
2501                fam_type                = &amd64_family_types[K8_CPUS];
2502                pvt->ops                = &amd64_family_types[K8_CPUS].ops;
2503                pvt->ctl_name           = fam_type->ctl_name;
2504                pvt->min_scrubrate      = K8_MIN_SCRUB_RATE_BITS;
2505                break;
2506        case 0x10:
2507                fam_type                = &amd64_family_types[F10_CPUS];
2508                pvt->ops                = &amd64_family_types[F10_CPUS].ops;
2509                pvt->ctl_name           = fam_type->ctl_name;
2510                pvt->min_scrubrate      = F10_MIN_SCRUB_RATE_BITS;
2511                break;
2512
2513        default:
2514                amd64_err("Unsupported family!\n");
2515                return NULL;
2516        }
2517
2518        pvt->ext_model = boot_cpu_data.x86_model >> 4;
2519
2520        amd64_info("%s %sdetected (node %d).\n", pvt->ctl_name,
2521                     (fam == 0xf ?
2522                                (pvt->ext_model >= K8_REV_F  ? "revF or later "
2523                                                             : "revE or earlier ")
2524                                 : ""), pvt->mc_node_id);
2525        return fam_type;
2526}
2527
2528static int amd64_init_one_instance(struct pci_dev *F2)
2529{
2530        struct amd64_pvt *pvt = NULL;
2531        struct amd64_family_type *fam_type = NULL;
2532        struct mem_ctl_info *mci = NULL;
2533        int err = 0, ret;
2534        u8 nid = get_node_id(F2);
2535
2536        ret = -ENOMEM;
2537        pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2538        if (!pvt)
2539                goto err_ret;
2540
2541        pvt->mc_node_id = nid;
2542        pvt->F2 = F2;
2543
2544        ret = -EINVAL;
2545        fam_type = amd64_per_family_init(pvt);
2546        if (!fam_type)
2547                goto err_free;
2548
2549        ret = -ENODEV;
2550        err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2551        if (err)
2552                goto err_free;
2553
2554        read_mc_regs(pvt);
2555
2556        /*
2557         * We need to determine how many memory channels there are. Then use
2558         * that information for calculating the size of the dynamic instance
2559         * tables in the 'mci' structure.
2560         */
2561        ret = -EINVAL;
2562        pvt->channel_count = pvt->ops->early_channel_count(pvt);
2563        if (pvt->channel_count < 0)
2564                goto err_siblings;
2565
2566        ret = -ENOMEM;
2567        mci = edac_mc_alloc(0, pvt->cs_count, pvt->channel_count, nid);
2568        if (!mci)
2569                goto err_siblings;
2570
2571        mci->pvt_info = pvt;
2572        mci->dev = &pvt->F2->dev;
2573
2574        setup_mci_misc_attrs(mci);
2575
2576        if (init_csrows(mci))
2577                mci->edac_cap = EDAC_FLAG_NONE;
2578
2579        set_mc_sysfs_attrs(mci);
2580
2581        ret = -ENODEV;
2582        if (edac_mc_add_mc(mci)) {
2583                debugf1("failed edac_mc_add_mc()\n");
2584                goto err_add_mc;
2585        }
2586
2587        /* register stuff with EDAC MCE */
2588        if (report_gart_errors)
2589                amd_report_gart_errors(true);
2590
2591        amd_register_ecc_decoder(amd64_decode_bus_error);
2592
2593        mcis[nid] = mci;
2594
2595        atomic_inc(&drv_instances);
2596
2597        return 0;
2598
2599err_add_mc:
2600        edac_mc_free(mci);
2601
2602err_siblings:
2603        free_mc_sibling_devs(pvt);
2604
2605err_free:
2606        kfree(pvt);
2607
2608err_ret:
2609        return ret;
2610}
2611
2612static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
2613                                             const struct pci_device_id *mc_type)
2614{
2615        u8 nid = get_node_id(pdev);
2616        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2617        struct ecc_settings *s;
2618        int ret = 0;
2619
2620        ret = pci_enable_device(pdev);
2621        if (ret < 0) {
2622                debugf0("ret=%d\n", ret);
2623                return -EIO;
2624        }
2625
2626        ret = -ENOMEM;
2627        s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2628        if (!s)
2629                goto err_out;
2630
2631        ecc_stngs[nid] = s;
2632
2633        if (!ecc_enabled(F3, nid)) {
2634                ret = -ENODEV;
2635
2636                if (!ecc_enable_override)
2637                        goto err_enable;
2638
2639                amd64_warn("Forcing ECC on!\n");
2640
2641                if (!enable_ecc_error_reporting(s, nid, F3))
2642                        goto err_enable;
2643        }
2644
2645        ret = amd64_init_one_instance(pdev);
2646        if (ret < 0) {
2647                amd64_err("Error probing instance: %d\n", nid);
2648                restore_ecc_error_reporting(s, nid, F3);
2649        }
2650
2651        return ret;
2652
2653err_enable:
2654        kfree(s);
2655        ecc_stngs[nid] = NULL;
2656
2657err_out:
2658        return ret;
2659}
2660
2661static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2662{
2663        struct mem_ctl_info *mci;
2664        struct amd64_pvt *pvt;
2665        u8 nid = get_node_id(pdev);
2666        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2667        struct ecc_settings *s = ecc_stngs[nid];
2668
2669        /* Remove from EDAC CORE tracking list */
2670        mci = edac_mc_del_mc(&pdev->dev);
2671        if (!mci)
2672                return;
2673
2674        pvt = mci->pvt_info;
2675
2676        restore_ecc_error_reporting(s, nid, F3);
2677
2678        free_mc_sibling_devs(pvt);
2679
2680        /* unregister from EDAC MCE */
2681        amd_report_gart_errors(false);
2682        amd_unregister_ecc_decoder(amd64_decode_bus_error);
2683
2684        kfree(ecc_stngs[nid]);
2685        ecc_stngs[nid] = NULL;
2686
2687        /* Free the EDAC CORE resources */
2688        mci->pvt_info = NULL;
2689        mcis[nid] = NULL;
2690
2691        kfree(pvt);
2692        edac_mc_free(mci);
2693}
2694
2695/*
2696 * This table is part of the interface for loading drivers for PCI devices. The
2697 * PCI core identifies what devices are on a system during boot, and then
2698 * inquiry this table to see if this driver is for a given device found.
2699 */
2700static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2701        {
2702                .vendor         = PCI_VENDOR_ID_AMD,
2703                .device         = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2704                .subvendor      = PCI_ANY_ID,
2705                .subdevice      = PCI_ANY_ID,
2706                .class          = 0,
2707                .class_mask     = 0,
2708        },
2709        {
2710                .vendor         = PCI_VENDOR_ID_AMD,
2711                .device         = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2712                .subvendor      = PCI_ANY_ID,
2713                .subdevice      = PCI_ANY_ID,
2714                .class          = 0,
2715                .class_mask     = 0,
2716        },
2717        {0, }
2718};
2719MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2720
2721static struct pci_driver amd64_pci_driver = {
2722        .name           = EDAC_MOD_STR,
2723        .probe          = amd64_probe_one_instance,
2724        .remove         = __devexit_p(amd64_remove_one_instance),
2725        .id_table       = amd64_pci_table,
2726};
2727
2728static void setup_pci_device(void)
2729{
2730        struct mem_ctl_info *mci;
2731        struct amd64_pvt *pvt;
2732
2733        if (amd64_ctl_pci)
2734                return;
2735
2736        mci = mcis[0];
2737        if (mci) {
2738
2739                pvt = mci->pvt_info;
2740                amd64_ctl_pci =
2741                        edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2742
2743                if (!amd64_ctl_pci) {
2744                        pr_warning("%s(): Unable to create PCI control\n",
2745                                   __func__);
2746
2747                        pr_warning("%s(): PCI error report via EDAC not set\n",
2748                                   __func__);
2749                        }
2750        }
2751}
2752
2753static int __init amd64_edac_init(void)
2754{
2755        int err = -ENODEV;
2756
2757        edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n");
2758
2759        opstate_init();
2760
2761        if (amd_cache_northbridges() < 0)
2762                goto err_ret;
2763
2764        err = -ENOMEM;
2765        mcis      = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2766        ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2767        if (!(mcis && ecc_stngs))
2768                goto err_free;
2769
2770        msrs = msrs_alloc();
2771        if (!msrs)
2772                goto err_free;
2773
2774        err = pci_register_driver(&amd64_pci_driver);
2775        if (err)
2776                goto err_pci;
2777
2778        err = -ENODEV;
2779        if (!atomic_read(&drv_instances))
2780                goto err_no_instances;
2781
2782        setup_pci_device();
2783        return 0;
2784
2785err_no_instances:
2786        pci_unregister_driver(&amd64_pci_driver);
2787
2788err_pci:
2789        msrs_free(msrs);
2790        msrs = NULL;
2791
2792err_free:
2793        kfree(mcis);
2794        mcis = NULL;
2795
2796        kfree(ecc_stngs);
2797        ecc_stngs = NULL;
2798
2799err_ret:
2800        return err;
2801}
2802
2803static void __exit amd64_edac_exit(void)
2804{
2805        if (amd64_ctl_pci)
2806                edac_pci_release_generic_ctl(amd64_ctl_pci);
2807
2808        pci_unregister_driver(&amd64_pci_driver);
2809
2810        kfree(ecc_stngs);
2811        ecc_stngs = NULL;
2812
2813        kfree(mcis);
2814        mcis = NULL;
2815
2816        msrs_free(msrs);
2817        msrs = NULL;
2818}
2819
2820module_init(amd64_edac_init);
2821module_exit(amd64_edac_exit);
2822
2823MODULE_LICENSE("GPL");
2824MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2825                "Dave Peterson, Thayne Harbaugh");
2826MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2827                EDAC_AMD64_VERSION);
2828
2829module_param(edac_op_state, int, 0444);
2830MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
2831