linux/drivers/edac/amd64_edac.c
<<
>>
Prefs
   1#include "amd64_edac.h"
   2#include <asm/amd_nb.h>
   3
   4static struct edac_pci_ctl_info *amd64_ctl_pci;
   5
   6static int report_gart_errors;
   7module_param(report_gart_errors, int, 0644);
   8
   9/*
  10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
  11 * cleared to prevent re-enabling the hardware by this driver.
  12 */
  13static int ecc_enable_override;
  14module_param(ecc_enable_override, int, 0644);
  15
  16static struct msr __percpu *msrs;
  17
  18/*
  19 * count successfully initialized driver instances for setup_pci_device()
  20 */
  21static atomic_t drv_instances = ATOMIC_INIT(0);
  22
  23/* Per-node driver instances */
  24static struct mem_ctl_info **mcis;
  25static struct ecc_settings **ecc_stngs;
  26
  27/*
  28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
  29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
  30 * or higher value'.
  31 *
  32 *FIXME: Produce a better mapping/linearisation.
  33 */
  34static const struct scrubrate {
  35       u32 scrubval;           /* bit pattern for scrub rate */
  36       u32 bandwidth;          /* bandwidth consumed (bytes/sec) */
  37} scrubrates[] = {
  38        { 0x01, 1600000000UL},
  39        { 0x02, 800000000UL},
  40        { 0x03, 400000000UL},
  41        { 0x04, 200000000UL},
  42        { 0x05, 100000000UL},
  43        { 0x06, 50000000UL},
  44        { 0x07, 25000000UL},
  45        { 0x08, 12284069UL},
  46        { 0x09, 6274509UL},
  47        { 0x0A, 3121951UL},
  48        { 0x0B, 1560975UL},
  49        { 0x0C, 781440UL},
  50        { 0x0D, 390720UL},
  51        { 0x0E, 195300UL},
  52        { 0x0F, 97650UL},
  53        { 0x10, 48854UL},
  54        { 0x11, 24427UL},
  55        { 0x12, 12213UL},
  56        { 0x13, 6101UL},
  57        { 0x14, 3051UL},
  58        { 0x15, 1523UL},
  59        { 0x16, 761UL},
  60        { 0x00, 0UL},        /* scrubbing off */
  61};
  62
  63int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
  64                               u32 *val, const char *func)
  65{
  66        int err = 0;
  67
  68        err = pci_read_config_dword(pdev, offset, val);
  69        if (err)
  70                amd64_warn("%s: error reading F%dx%03x.\n",
  71                           func, PCI_FUNC(pdev->devfn), offset);
  72
  73        return err;
  74}
  75
  76int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
  77                                u32 val, const char *func)
  78{
  79        int err = 0;
  80
  81        err = pci_write_config_dword(pdev, offset, val);
  82        if (err)
  83                amd64_warn("%s: error writing to F%dx%03x.\n",
  84                           func, PCI_FUNC(pdev->devfn), offset);
  85
  86        return err;
  87}
  88
  89/*
  90 *
  91 * Depending on the family, F2 DCT reads need special handling:
  92 *
  93 * K8: has a single DCT only
  94 *
  95 * F10h: each DCT has its own set of regs
  96 *      DCT0 -> F2x040..
  97 *      DCT1 -> F2x140..
  98 *
  99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
 100 *
 101 * F16h: has only 1 DCT
 102 */
 103static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
 104                               const char *func)
 105{
 106        if (addr >= 0x100)
 107                return -EINVAL;
 108
 109        return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
 110}
 111
 112static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
 113                                 const char *func)
 114{
 115        return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
 116}
 117
 118/*
 119 * Select DCT to which PCI cfg accesses are routed
 120 */
 121static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
 122{
 123        u32 reg = 0;
 124
 125        amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
 126        reg &= 0xfffffffe;
 127        reg |= dct;
 128        amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
 129}
 130
 131static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
 132                                 const char *func)
 133{
 134        u8 dct  = 0;
 135
 136        if (addr >= 0x140 && addr <= 0x1a0) {
 137                dct   = 1;
 138                addr -= 0x100;
 139        }
 140
 141        f15h_select_dct(pvt, dct);
 142
 143        return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
 144}
 145
 146/*
 147 * Memory scrubber control interface. For K8, memory scrubbing is handled by
 148 * hardware and can involve L2 cache, dcache as well as the main memory. With
 149 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
 150 * functionality.
 151 *
 152 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
 153 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
 154 * bytes/sec for the setting.
 155 *
 156 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
 157 * other archs, we might not have access to the caches directly.
 158 */
 159
 160/*
 161 * scan the scrub rate mapping table for a close or matching bandwidth value to
 162 * issue. If requested is too big, then use last maximum value found.
 163 */
 164static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
 165{
 166        u32 scrubval;
 167        int i;
 168
 169        /*
 170         * map the configured rate (new_bw) to a value specific to the AMD64
 171         * memory controller and apply to register. Search for the first
 172         * bandwidth entry that is greater or equal than the setting requested
 173         * and program that. If at last entry, turn off DRAM scrubbing.
 174         *
 175         * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
 176         * by falling back to the last element in scrubrates[].
 177         */
 178        for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
 179                /*
 180                 * skip scrub rates which aren't recommended
 181                 * (see F10 BKDG, F3x58)
 182                 */
 183                if (scrubrates[i].scrubval < min_rate)
 184                        continue;
 185
 186                if (scrubrates[i].bandwidth <= new_bw)
 187                        break;
 188        }
 189
 190        scrubval = scrubrates[i].scrubval;
 191
 192        pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
 193
 194        if (scrubval)
 195                return scrubrates[i].bandwidth;
 196
 197        return 0;
 198}
 199
 200static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
 201{
 202        struct amd64_pvt *pvt = mci->pvt_info;
 203        u32 min_scrubrate = 0x5;
 204
 205        if (boot_cpu_data.x86 == 0xf)
 206                min_scrubrate = 0x0;
 207
 208        /* F15h Erratum #505 */
 209        if (boot_cpu_data.x86 == 0x15)
 210                f15h_select_dct(pvt, 0);
 211
 212        return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
 213}
 214
 215static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
 216{
 217        struct amd64_pvt *pvt = mci->pvt_info;
 218        u32 scrubval = 0;
 219        int i, retval = -EINVAL;
 220
 221        /* F15h Erratum #505 */
 222        if (boot_cpu_data.x86 == 0x15)
 223                f15h_select_dct(pvt, 0);
 224
 225        amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
 226
 227        scrubval = scrubval & 0x001F;
 228
 229        for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
 230                if (scrubrates[i].scrubval == scrubval) {
 231                        retval = scrubrates[i].bandwidth;
 232                        break;
 233                }
 234        }
 235        return retval;
 236}
 237
 238/*
 239 * returns true if the SysAddr given by sys_addr matches the
 240 * DRAM base/limit associated with node_id
 241 */
 242static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
 243                                   u8 nid)
 244{
 245        u64 addr;
 246
 247        /* The K8 treats this as a 40-bit value.  However, bits 63-40 will be
 248         * all ones if the most significant implemented address bit is 1.
 249         * Here we discard bits 63-40.  See section 3.4.2 of AMD publication
 250         * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
 251         * Application Programming.
 252         */
 253        addr = sys_addr & 0x000000ffffffffffull;
 254
 255        return ((addr >= get_dram_base(pvt, nid)) &&
 256                (addr <= get_dram_limit(pvt, nid)));
 257}
 258
 259/*
 260 * Attempt to map a SysAddr to a node. On success, return a pointer to the
 261 * mem_ctl_info structure for the node that the SysAddr maps to.
 262 *
 263 * On failure, return NULL.
 264 */
 265static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
 266                                                u64 sys_addr)
 267{
 268        struct amd64_pvt *pvt;
 269        u8 node_id;
 270        u32 intlv_en, bits;
 271
 272        /*
 273         * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
 274         * 3.4.4.2) registers to map the SysAddr to a node ID.
 275         */
 276        pvt = mci->pvt_info;
 277
 278        /*
 279         * The value of this field should be the same for all DRAM Base
 280         * registers.  Therefore we arbitrarily choose to read it from the
 281         * register for node 0.
 282         */
 283        intlv_en = dram_intlv_en(pvt, 0);
 284
 285        if (intlv_en == 0) {
 286                for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
 287                        if (amd64_base_limit_match(pvt, sys_addr, node_id))
 288                                goto found;
 289                }
 290                goto err_no_match;
 291        }
 292
 293        if (unlikely((intlv_en != 0x01) &&
 294                     (intlv_en != 0x03) &&
 295                     (intlv_en != 0x07))) {
 296                amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
 297                return NULL;
 298        }
 299
 300        bits = (((u32) sys_addr) >> 12) & intlv_en;
 301
 302        for (node_id = 0; ; ) {
 303                if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
 304                        break;  /* intlv_sel field matches */
 305
 306                if (++node_id >= DRAM_RANGES)
 307                        goto err_no_match;
 308        }
 309
 310        /* sanity test for sys_addr */
 311        if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
 312                amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
 313                           "range for node %d with node interleaving enabled.\n",
 314                           __func__, sys_addr, node_id);
 315                return NULL;
 316        }
 317
 318found:
 319        return edac_mc_find((int)node_id);
 320
 321err_no_match:
 322        edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
 323                 (unsigned long)sys_addr);
 324
 325        return NULL;
 326}
 327
 328/*
 329 * compute the CS base address of the @csrow on the DRAM controller @dct.
 330 * For details see F2x[5C:40] in the processor's BKDG
 331 */
 332static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
 333                                 u64 *base, u64 *mask)
 334{
 335        u64 csbase, csmask, base_bits, mask_bits;
 336        u8 addr_shift;
 337
 338        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
 339                csbase          = pvt->csels[dct].csbases[csrow];
 340                csmask          = pvt->csels[dct].csmasks[csrow];
 341                base_bits       = GENMASK(21, 31) | GENMASK(9, 15);
 342                mask_bits       = GENMASK(21, 29) | GENMASK(9, 15);
 343                addr_shift      = 4;
 344
 345        /*
 346        * F16h needs two addr_shift values: 8 for high and 6 for low
 347        * (cf. F16h BKDG).
 348        */
 349        } else if (boot_cpu_data.x86 == 0x16) {
 350                csbase          = pvt->csels[dct].csbases[csrow];
 351                csmask          = pvt->csels[dct].csmasks[csrow >> 1];
 352
 353                *base  = (csbase & GENMASK(5,  15)) << 6;
 354                *base |= (csbase & GENMASK(19, 30)) << 8;
 355
 356                *mask = ~0ULL;
 357                /* poke holes for the csmask */
 358                *mask &= ~((GENMASK(5, 15)  << 6) |
 359                           (GENMASK(19, 30) << 8));
 360
 361                *mask |= (csmask & GENMASK(5, 15))  << 6;
 362                *mask |= (csmask & GENMASK(19, 30)) << 8;
 363
 364                return;
 365        } else {
 366                csbase          = pvt->csels[dct].csbases[csrow];
 367                csmask          = pvt->csels[dct].csmasks[csrow >> 1];
 368                addr_shift      = 8;
 369
 370                if (boot_cpu_data.x86 == 0x15)
 371                        base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
 372                else
 373                        base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
 374        }
 375
 376        *base  = (csbase & base_bits) << addr_shift;
 377
 378        *mask  = ~0ULL;
 379        /* poke holes for the csmask */
 380        *mask &= ~(mask_bits << addr_shift);
 381        /* OR them in */
 382        *mask |= (csmask & mask_bits) << addr_shift;
 383}
 384
 385#define for_each_chip_select(i, dct, pvt) \
 386        for (i = 0; i < pvt->csels[dct].b_cnt; i++)
 387
 388#define chip_select_base(i, dct, pvt) \
 389        pvt->csels[dct].csbases[i]
 390
 391#define for_each_chip_select_mask(i, dct, pvt) \
 392        for (i = 0; i < pvt->csels[dct].m_cnt; i++)
 393
 394/*
 395 * @input_addr is an InputAddr associated with the node given by mci. Return the
 396 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
 397 */
 398static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
 399{
 400        struct amd64_pvt *pvt;
 401        int csrow;
 402        u64 base, mask;
 403
 404        pvt = mci->pvt_info;
 405
 406        for_each_chip_select(csrow, 0, pvt) {
 407                if (!csrow_enabled(csrow, 0, pvt))
 408                        continue;
 409
 410                get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
 411
 412                mask = ~mask;
 413
 414                if ((input_addr & mask) == (base & mask)) {
 415                        edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
 416                                 (unsigned long)input_addr, csrow,
 417                                 pvt->mc_node_id);
 418
 419                        return csrow;
 420                }
 421        }
 422        edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
 423                 (unsigned long)input_addr, pvt->mc_node_id);
 424
 425        return -1;
 426}
 427
 428/*
 429 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
 430 * for the node represented by mci. Info is passed back in *hole_base,
 431 * *hole_offset, and *hole_size.  Function returns 0 if info is valid or 1 if
 432 * info is invalid. Info may be invalid for either of the following reasons:
 433 *
 434 * - The revision of the node is not E or greater.  In this case, the DRAM Hole
 435 *   Address Register does not exist.
 436 *
 437 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
 438 *   indicating that its contents are not valid.
 439 *
 440 * The values passed back in *hole_base, *hole_offset, and *hole_size are
 441 * complete 32-bit values despite the fact that the bitfields in the DHAR
 442 * only represent bits 31-24 of the base and offset values.
 443 */
 444int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
 445                             u64 *hole_offset, u64 *hole_size)
 446{
 447        struct amd64_pvt *pvt = mci->pvt_info;
 448
 449        /* only revE and later have the DRAM Hole Address Register */
 450        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
 451                edac_dbg(1, "  revision %d for node %d does not support DHAR\n",
 452                         pvt->ext_model, pvt->mc_node_id);
 453                return 1;
 454        }
 455
 456        /* valid for Fam10h and above */
 457        if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
 458                edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this system\n");
 459                return 1;
 460        }
 461
 462        if (!dhar_valid(pvt)) {
 463                edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this node %d\n",
 464                         pvt->mc_node_id);
 465                return 1;
 466        }
 467
 468        /* This node has Memory Hoisting */
 469
 470        /* +------------------+--------------------+--------------------+-----
 471         * | memory           | DRAM hole          | relocated          |
 472         * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
 473         * |                  |                    | DRAM hole          |
 474         * |                  |                    | [0x100000000,      |
 475         * |                  |                    |  (0x100000000+     |
 476         * |                  |                    |   (0xffffffff-x))] |
 477         * +------------------+--------------------+--------------------+-----
 478         *
 479         * Above is a diagram of physical memory showing the DRAM hole and the
 480         * relocated addresses from the DRAM hole.  As shown, the DRAM hole
 481         * starts at address x (the base address) and extends through address
 482         * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
 483         * addresses in the hole so that they start at 0x100000000.
 484         */
 485
 486        *hole_base = dhar_base(pvt);
 487        *hole_size = (1ULL << 32) - *hole_base;
 488
 489        if (boot_cpu_data.x86 > 0xf)
 490                *hole_offset = f10_dhar_offset(pvt);
 491        else
 492                *hole_offset = k8_dhar_offset(pvt);
 493
 494        edac_dbg(1, "  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
 495                 pvt->mc_node_id, (unsigned long)*hole_base,
 496                 (unsigned long)*hole_offset, (unsigned long)*hole_size);
 497
 498        return 0;
 499}
 500EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
 501
 502/*
 503 * Return the DramAddr that the SysAddr given by @sys_addr maps to.  It is
 504 * assumed that sys_addr maps to the node given by mci.
 505 *
 506 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
 507 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
 508 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
 509 * then it is also involved in translating a SysAddr to a DramAddr. Sections
 510 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
 511 * These parts of the documentation are unclear. I interpret them as follows:
 512 *
 513 * When node n receives a SysAddr, it processes the SysAddr as follows:
 514 *
 515 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
 516 *    Limit registers for node n. If the SysAddr is not within the range
 517 *    specified by the base and limit values, then node n ignores the Sysaddr
 518 *    (since it does not map to node n). Otherwise continue to step 2 below.
 519 *
 520 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
 521 *    disabled so skip to step 3 below. Otherwise see if the SysAddr is within
 522 *    the range of relocated addresses (starting at 0x100000000) from the DRAM
 523 *    hole. If not, skip to step 3 below. Else get the value of the
 524 *    DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
 525 *    offset defined by this value from the SysAddr.
 526 *
 527 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
 528 *    Base register for node n. To obtain the DramAddr, subtract the base
 529 *    address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
 530 */
 531static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
 532{
 533        struct amd64_pvt *pvt = mci->pvt_info;
 534        u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
 535        int ret;
 536
 537        dram_base = get_dram_base(pvt, pvt->mc_node_id);
 538
 539        ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
 540                                      &hole_size);
 541        if (!ret) {
 542                if ((sys_addr >= (1ULL << 32)) &&
 543                    (sys_addr < ((1ULL << 32) + hole_size))) {
 544                        /* use DHAR to translate SysAddr to DramAddr */
 545                        dram_addr = sys_addr - hole_offset;
 546
 547                        edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
 548                                 (unsigned long)sys_addr,
 549                                 (unsigned long)dram_addr);
 550
 551                        return dram_addr;
 552                }
 553        }
 554
 555        /*
 556         * Translate the SysAddr to a DramAddr as shown near the start of
 557         * section 3.4.4 (p. 70).  Although sys_addr is a 64-bit value, the k8
 558         * only deals with 40-bit values.  Therefore we discard bits 63-40 of
 559         * sys_addr below.  If bit 39 of sys_addr is 1 then the bits we
 560         * discard are all 1s.  Otherwise the bits we discard are all 0s.  See
 561         * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
 562         * Programmer's Manual Volume 1 Application Programming.
 563         */
 564        dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
 565
 566        edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
 567                 (unsigned long)sys_addr, (unsigned long)dram_addr);
 568        return dram_addr;
 569}
 570
 571/*
 572 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
 573 * (section 3.4.4.1).  Return the number of bits from a SysAddr that are used
 574 * for node interleaving.
 575 */
 576static int num_node_interleave_bits(unsigned intlv_en)
 577{
 578        static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
 579        int n;
 580
 581        BUG_ON(intlv_en > 7);
 582        n = intlv_shift_table[intlv_en];
 583        return n;
 584}
 585
 586/* Translate the DramAddr given by @dram_addr to an InputAddr. */
 587static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
 588{
 589        struct amd64_pvt *pvt;
 590        int intlv_shift;
 591        u64 input_addr;
 592
 593        pvt = mci->pvt_info;
 594
 595        /*
 596         * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
 597         * concerning translating a DramAddr to an InputAddr.
 598         */
 599        intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
 600        input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
 601                      (dram_addr & 0xfff);
 602
 603        edac_dbg(2, "  Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
 604                 intlv_shift, (unsigned long)dram_addr,
 605                 (unsigned long)input_addr);
 606
 607        return input_addr;
 608}
 609
 610/*
 611 * Translate the SysAddr represented by @sys_addr to an InputAddr.  It is
 612 * assumed that @sys_addr maps to the node given by mci.
 613 */
 614static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
 615{
 616        u64 input_addr;
 617
 618        input_addr =
 619            dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
 620
 621        edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
 622                 (unsigned long)sys_addr, (unsigned long)input_addr);
 623
 624        return input_addr;
 625}
 626
 627/* Map the Error address to a PAGE and PAGE OFFSET. */
 628static inline void error_address_to_page_and_offset(u64 error_address,
 629                                                    struct err_info *err)
 630{
 631        err->page = (u32) (error_address >> PAGE_SHIFT);
 632        err->offset = ((u32) error_address) & ~PAGE_MASK;
 633}
 634
 635/*
 636 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
 637 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
 638 * of a node that detected an ECC memory error.  mci represents the node that
 639 * the error address maps to (possibly different from the node that detected
 640 * the error).  Return the number of the csrow that sys_addr maps to, or -1 on
 641 * error.
 642 */
 643static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
 644{
 645        int csrow;
 646
 647        csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
 648
 649        if (csrow == -1)
 650                amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
 651                                  "address 0x%lx\n", (unsigned long)sys_addr);
 652        return csrow;
 653}
 654
 655static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
 656
 657/*
 658 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
 659 * are ECC capable.
 660 */
 661static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
 662{
 663        u8 bit;
 664        unsigned long edac_cap = EDAC_FLAG_NONE;
 665
 666        bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
 667                ? 19
 668                : 17;
 669
 670        if (pvt->dclr0 & BIT(bit))
 671                edac_cap = EDAC_FLAG_SECDED;
 672
 673        return edac_cap;
 674}
 675
 676static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
 677
 678static void amd64_dump_dramcfg_low(u32 dclr, int chan)
 679{
 680        edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
 681
 682        edac_dbg(1, "  DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
 683                 (dclr & BIT(16)) ?  "un" : "",
 684                 (dclr & BIT(19)) ? "yes" : "no");
 685
 686        edac_dbg(1, "  PAR/ERR parity: %s\n",
 687                 (dclr & BIT(8)) ?  "enabled" : "disabled");
 688
 689        if (boot_cpu_data.x86 == 0x10)
 690                edac_dbg(1, "  DCT 128bit mode width: %s\n",
 691                         (dclr & BIT(11)) ?  "128b" : "64b");
 692
 693        edac_dbg(1, "  x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
 694                 (dclr & BIT(12)) ?  "yes" : "no",
 695                 (dclr & BIT(13)) ?  "yes" : "no",
 696                 (dclr & BIT(14)) ?  "yes" : "no",
 697                 (dclr & BIT(15)) ?  "yes" : "no");
 698}
 699
 700/* Display and decode various NB registers for debug purposes. */
 701static void dump_misc_regs(struct amd64_pvt *pvt)
 702{
 703        edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
 704
 705        edac_dbg(1, "  NB two channel DRAM capable: %s\n",
 706                 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
 707
 708        edac_dbg(1, "  ECC capable: %s, ChipKill ECC capable: %s\n",
 709                 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
 710                 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
 711
 712        amd64_dump_dramcfg_low(pvt->dclr0, 0);
 713
 714        edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
 715
 716        edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
 717                 pvt->dhar, dhar_base(pvt),
 718                 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
 719                 : f10_dhar_offset(pvt));
 720
 721        edac_dbg(1, "  DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
 722
 723        amd64_debug_display_dimm_sizes(pvt, 0);
 724
 725        /* everything below this point is Fam10h and above */
 726        if (boot_cpu_data.x86 == 0xf)
 727                return;
 728
 729        amd64_debug_display_dimm_sizes(pvt, 1);
 730
 731        amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
 732
 733        /* Only if NOT ganged does dclr1 have valid info */
 734        if (!dct_ganging_enabled(pvt))
 735                amd64_dump_dramcfg_low(pvt->dclr1, 1);
 736}
 737
 738/*
 739 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
 740 */
 741static void prep_chip_selects(struct amd64_pvt *pvt)
 742{
 743        if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
 744                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
 745                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
 746        } else {
 747                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
 748                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
 749        }
 750}
 751
 752/*
 753 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
 754 */
 755static void read_dct_base_mask(struct amd64_pvt *pvt)
 756{
 757        int cs;
 758
 759        prep_chip_selects(pvt);
 760
 761        for_each_chip_select(cs, 0, pvt) {
 762                int reg0   = DCSB0 + (cs * 4);
 763                int reg1   = DCSB1 + (cs * 4);
 764                u32 *base0 = &pvt->csels[0].csbases[cs];
 765                u32 *base1 = &pvt->csels[1].csbases[cs];
 766
 767                if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
 768                        edac_dbg(0, "  DCSB0[%d]=0x%08x reg: F2x%x\n",
 769                                 cs, *base0, reg0);
 770
 771                if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
 772                        continue;
 773
 774                if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
 775                        edac_dbg(0, "  DCSB1[%d]=0x%08x reg: F2x%x\n",
 776                                 cs, *base1, reg1);
 777        }
 778
 779        for_each_chip_select_mask(cs, 0, pvt) {
 780                int reg0   = DCSM0 + (cs * 4);
 781                int reg1   = DCSM1 + (cs * 4);
 782                u32 *mask0 = &pvt->csels[0].csmasks[cs];
 783                u32 *mask1 = &pvt->csels[1].csmasks[cs];
 784
 785                if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
 786                        edac_dbg(0, "    DCSM0[%d]=0x%08x reg: F2x%x\n",
 787                                 cs, *mask0, reg0);
 788
 789                if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
 790                        continue;
 791
 792                if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
 793                        edac_dbg(0, "    DCSM1[%d]=0x%08x reg: F2x%x\n",
 794                                 cs, *mask1, reg1);
 795        }
 796}
 797
 798static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
 799{
 800        enum mem_type type;
 801
 802        /* F15h supports only DDR3 */
 803        if (boot_cpu_data.x86 >= 0x15)
 804                type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
 805        else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
 806                if (pvt->dchr0 & DDR3_MODE)
 807                        type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
 808                else
 809                        type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
 810        } else {
 811                type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
 812        }
 813
 814        amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
 815
 816        return type;
 817}
 818
 819/* Get the number of DCT channels the memory controller is using. */
 820static int k8_early_channel_count(struct amd64_pvt *pvt)
 821{
 822        int flag;
 823
 824        if (pvt->ext_model >= K8_REV_F)
 825                /* RevF (NPT) and later */
 826                flag = pvt->dclr0 & WIDTH_128;
 827        else
 828                /* RevE and earlier */
 829                flag = pvt->dclr0 & REVE_WIDTH_128;
 830
 831        /* not used */
 832        pvt->dclr1 = 0;
 833
 834        return (flag) ? 2 : 1;
 835}
 836
 837/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
 838static u64 get_error_address(struct mce *m)
 839{
 840        struct cpuinfo_x86 *c = &boot_cpu_data;
 841        u64 addr;
 842        u8 start_bit = 1;
 843        u8 end_bit   = 47;
 844
 845        if (c->x86 == 0xf) {
 846                start_bit = 3;
 847                end_bit   = 39;
 848        }
 849
 850        addr = m->addr & GENMASK(start_bit, end_bit);
 851
 852        /*
 853         * Erratum 637 workaround
 854         */
 855        if (c->x86 == 0x15) {
 856                struct amd64_pvt *pvt;
 857                u64 cc6_base, tmp_addr;
 858                u32 tmp;
 859                u16 mce_nid;
 860                u8 intlv_en;
 861
 862                if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
 863                        return addr;
 864
 865                mce_nid = amd_get_nb_id(m->extcpu);
 866                pvt     = mcis[mce_nid]->pvt_info;
 867
 868                amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
 869                intlv_en = tmp >> 21 & 0x7;
 870
 871                /* add [47:27] + 3 trailing bits */
 872                cc6_base  = (tmp & GENMASK(0, 20)) << 3;
 873
 874                /* reverse and add DramIntlvEn */
 875                cc6_base |= intlv_en ^ 0x7;
 876
 877                /* pin at [47:24] */
 878                cc6_base <<= 24;
 879
 880                if (!intlv_en)
 881                        return cc6_base | (addr & GENMASK(0, 23));
 882
 883                amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
 884
 885                                                        /* faster log2 */
 886                tmp_addr  = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
 887
 888                /* OR DramIntlvSel into bits [14:12] */
 889                tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
 890
 891                /* add remaining [11:0] bits from original MC4_ADDR */
 892                tmp_addr |= addr & GENMASK(0, 11);
 893
 894                return cc6_base | tmp_addr;
 895        }
 896
 897        return addr;
 898}
 899
 900static struct pci_dev *pci_get_related_function(unsigned int vendor,
 901                                                unsigned int device,
 902                                                struct pci_dev *related)
 903{
 904        struct pci_dev *dev = NULL;
 905
 906        while ((dev = pci_get_device(vendor, device, dev))) {
 907                if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
 908                    (dev->bus->number == related->bus->number) &&
 909                    (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
 910                        break;
 911        }
 912
 913        return dev;
 914}
 915
 916static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
 917{
 918        struct amd_northbridge *nb;
 919        struct pci_dev *misc, *f1 = NULL;
 920        struct cpuinfo_x86 *c = &boot_cpu_data;
 921        int off = range << 3;
 922        u32 llim;
 923
 924        amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off,  &pvt->ranges[range].base.lo);
 925        amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
 926
 927        if (c->x86 == 0xf)
 928                return;
 929
 930        if (!dram_rw(pvt, range))
 931                return;
 932
 933        amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off,  &pvt->ranges[range].base.hi);
 934        amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
 935
 936        /* F15h: factor in CC6 save area by reading dst node's limit reg */
 937        if (c->x86 != 0x15)
 938                return;
 939
 940        nb = node_to_amd_nb(dram_dst_node(pvt, range));
 941        if (WARN_ON(!nb))
 942                return;
 943
 944        misc = nb->misc;
 945        f1 = pci_get_related_function(misc->vendor, PCI_DEVICE_ID_AMD_15H_NB_F1, misc);
 946        if (WARN_ON(!f1))
 947                return;
 948
 949        amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
 950
 951        pvt->ranges[range].lim.lo &= GENMASK(0, 15);
 952
 953                                    /* {[39:27],111b} */
 954        pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
 955
 956        pvt->ranges[range].lim.hi &= GENMASK(0, 7);
 957
 958                                    /* [47:40] */
 959        pvt->ranges[range].lim.hi |= llim >> 13;
 960
 961        pci_dev_put(f1);
 962}
 963
 964static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
 965                                    struct err_info *err)
 966{
 967        struct amd64_pvt *pvt = mci->pvt_info;
 968
 969        error_address_to_page_and_offset(sys_addr, err);
 970
 971        /*
 972         * Find out which node the error address belongs to. This may be
 973         * different from the node that detected the error.
 974         */
 975        err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
 976        if (!err->src_mci) {
 977                amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
 978                             (unsigned long)sys_addr);
 979                err->err_code = ERR_NODE;
 980                return;
 981        }
 982
 983        /* Now map the sys_addr to a CSROW */
 984        err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
 985        if (err->csrow < 0) {
 986                err->err_code = ERR_CSROW;
 987                return;
 988        }
 989
 990        /* CHIPKILL enabled */
 991        if (pvt->nbcfg & NBCFG_CHIPKILL) {
 992                err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
 993                if (err->channel < 0) {
 994                        /*
 995                         * Syndrome didn't map, so we don't know which of the
 996                         * 2 DIMMs is in error. So we need to ID 'both' of them
 997                         * as suspect.
 998                         */
 999                        amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1000                                      "possible error reporting race\n",
1001                                      err->syndrome);
1002                        err->err_code = ERR_CHANNEL;
1003                        return;
1004                }
1005        } else {
1006                /*
1007                 * non-chipkill ecc mode
1008                 *
1009                 * The k8 documentation is unclear about how to determine the
1010                 * channel number when using non-chipkill memory.  This method
1011                 * was obtained from email communication with someone at AMD.
1012                 * (Wish the email was placed in this comment - norsk)
1013                 */
1014                err->channel = ((sys_addr & BIT(3)) != 0);
1015        }
1016}
1017
1018static int ddr2_cs_size(unsigned i, bool dct_width)
1019{
1020        unsigned shift = 0;
1021
1022        if (i <= 2)
1023                shift = i;
1024        else if (!(i & 0x1))
1025                shift = i >> 1;
1026        else
1027                shift = (i + 1) >> 1;
1028
1029        return 128 << (shift + !!dct_width);
1030}
1031
1032static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1033                                  unsigned cs_mode)
1034{
1035        u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1036
1037        if (pvt->ext_model >= K8_REV_F) {
1038                WARN_ON(cs_mode > 11);
1039                return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1040        }
1041        else if (pvt->ext_model >= K8_REV_D) {
1042                unsigned diff;
1043                WARN_ON(cs_mode > 10);
1044
1045                /*
1046                 * the below calculation, besides trying to win an obfuscated C
1047                 * contest, maps cs_mode values to DIMM chip select sizes. The
1048                 * mappings are:
1049                 *
1050                 * cs_mode      CS size (mb)
1051                 * =======      ============
1052                 * 0            32
1053                 * 1            64
1054                 * 2            128
1055                 * 3            128
1056                 * 4            256
1057                 * 5            512
1058                 * 6            256
1059                 * 7            512
1060                 * 8            1024
1061                 * 9            1024
1062                 * 10           2048
1063                 *
1064                 * Basically, it calculates a value with which to shift the
1065                 * smallest CS size of 32MB.
1066                 *
1067                 * ddr[23]_cs_size have a similar purpose.
1068                 */
1069                diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1070
1071                return 32 << (cs_mode - diff);
1072        }
1073        else {
1074                WARN_ON(cs_mode > 6);
1075                return 32 << cs_mode;
1076        }
1077}
1078
1079/*
1080 * Get the number of DCT channels in use.
1081 *
1082 * Return:
1083 *      number of Memory Channels in operation
1084 * Pass back:
1085 *      contents of the DCL0_LOW register
1086 */
1087static int f1x_early_channel_count(struct amd64_pvt *pvt)
1088{
1089        int i, j, channels = 0;
1090
1091        /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1092        if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1093                return 2;
1094
1095        /*
1096         * Need to check if in unganged mode: In such, there are 2 channels,
1097         * but they are not in 128 bit mode and thus the above 'dclr0' status
1098         * bit will be OFF.
1099         *
1100         * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1101         * their CSEnable bit on. If so, then SINGLE DIMM case.
1102         */
1103        edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1104
1105        /*
1106         * Check DRAM Bank Address Mapping values for each DIMM to see if there
1107         * is more than just one DIMM present in unganged mode. Need to check
1108         * both controllers since DIMMs can be placed in either one.
1109         */
1110        for (i = 0; i < 2; i++) {
1111                u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1112
1113                for (j = 0; j < 4; j++) {
1114                        if (DBAM_DIMM(j, dbam) > 0) {
1115                                channels++;
1116                                break;
1117                        }
1118                }
1119        }
1120
1121        if (channels > 2)
1122                channels = 2;
1123
1124        amd64_info("MCT channel count: %d\n", channels);
1125
1126        return channels;
1127}
1128
1129static int ddr3_cs_size(unsigned i, bool dct_width)
1130{
1131        unsigned shift = 0;
1132        int cs_size = 0;
1133
1134        if (i == 0 || i == 3 || i == 4)
1135                cs_size = -1;
1136        else if (i <= 2)
1137                shift = i;
1138        else if (i == 12)
1139                shift = 7;
1140        else if (!(i & 0x1))
1141                shift = i >> 1;
1142        else
1143                shift = (i + 1) >> 1;
1144
1145        if (cs_size != -1)
1146                cs_size = (128 * (1 << !!dct_width)) << shift;
1147
1148        return cs_size;
1149}
1150
1151static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1152                                   unsigned cs_mode)
1153{
1154        u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1155
1156        WARN_ON(cs_mode > 11);
1157
1158        if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1159                return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1160        else
1161                return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1162}
1163
1164/*
1165 * F15h supports only 64bit DCT interfaces
1166 */
1167static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1168                                   unsigned cs_mode)
1169{
1170        WARN_ON(cs_mode > 12);
1171
1172        return ddr3_cs_size(cs_mode, false);
1173}
1174
1175/*
1176 * F16h has only limited cs_modes
1177 */
1178static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1179                                unsigned cs_mode)
1180{
1181        WARN_ON(cs_mode > 12);
1182
1183        if (cs_mode == 6 || cs_mode == 8 ||
1184            cs_mode == 9 || cs_mode == 12)
1185                return -1;
1186        else
1187                return ddr3_cs_size(cs_mode, false);
1188}
1189
1190static void read_dram_ctl_register(struct amd64_pvt *pvt)
1191{
1192
1193        if (boot_cpu_data.x86 == 0xf)
1194                return;
1195
1196        if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1197                edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1198                         pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1199
1200                edac_dbg(0, "  DCTs operate in %s mode\n",
1201                         (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1202
1203                if (!dct_ganging_enabled(pvt))
1204                        edac_dbg(0, "  Address range split per DCT: %s\n",
1205                                 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1206
1207                edac_dbg(0, "  data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1208                         (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1209                         (dct_memory_cleared(pvt) ? "yes" : "no"));
1210
1211                edac_dbg(0, "  channel interleave: %s, "
1212                         "interleave bits selector: 0x%x\n",
1213                         (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1214                         dct_sel_interleave_addr(pvt));
1215        }
1216
1217        amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1218}
1219
1220/*
1221 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1222 * Interleaving Modes.
1223 */
1224static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1225                                bool hi_range_sel, u8 intlv_en)
1226{
1227        u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1228
1229        if (dct_ganging_enabled(pvt))
1230                return 0;
1231
1232        if (hi_range_sel)
1233                return dct_sel_high;
1234
1235        /*
1236         * see F2x110[DctSelIntLvAddr] - channel interleave mode
1237         */
1238        if (dct_interleave_enabled(pvt)) {
1239                u8 intlv_addr = dct_sel_interleave_addr(pvt);
1240
1241                /* return DCT select function: 0=DCT0, 1=DCT1 */
1242                if (!intlv_addr)
1243                        return sys_addr >> 6 & 1;
1244
1245                if (intlv_addr & 0x2) {
1246                        u8 shift = intlv_addr & 0x1 ? 9 : 6;
1247                        u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1248
1249                        return ((sys_addr >> shift) & 1) ^ temp;
1250                }
1251
1252                return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1253        }
1254
1255        if (dct_high_range_enabled(pvt))
1256                return ~dct_sel_high & 1;
1257
1258        return 0;
1259}
1260
1261/* Convert the sys_addr to the normalized DCT address */
1262static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1263                                 u64 sys_addr, bool hi_rng,
1264                                 u32 dct_sel_base_addr)
1265{
1266        u64 chan_off;
1267        u64 dram_base           = get_dram_base(pvt, range);
1268        u64 hole_off            = f10_dhar_offset(pvt);
1269        u64 dct_sel_base_off    = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1270
1271        if (hi_rng) {
1272                /*
1273                 * if
1274                 * base address of high range is below 4Gb
1275                 * (bits [47:27] at [31:11])
1276                 * DRAM address space on this DCT is hoisted above 4Gb  &&
1277                 * sys_addr > 4Gb
1278                 *
1279                 *      remove hole offset from sys_addr
1280                 * else
1281                 *      remove high range offset from sys_addr
1282                 */
1283                if ((!(dct_sel_base_addr >> 16) ||
1284                     dct_sel_base_addr < dhar_base(pvt)) &&
1285                    dhar_valid(pvt) &&
1286                    (sys_addr >= BIT_64(32)))
1287                        chan_off = hole_off;
1288                else
1289                        chan_off = dct_sel_base_off;
1290        } else {
1291                /*
1292                 * if
1293                 * we have a valid hole         &&
1294                 * sys_addr > 4Gb
1295                 *
1296                 *      remove hole
1297                 * else
1298                 *      remove dram base to normalize to DCT address
1299                 */
1300                if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1301                        chan_off = hole_off;
1302                else
1303                        chan_off = dram_base;
1304        }
1305
1306        return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1307}
1308
1309/*
1310 * checks if the csrow passed in is marked as SPARED, if so returns the new
1311 * spare row
1312 */
1313static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1314{
1315        int tmp_cs;
1316
1317        if (online_spare_swap_done(pvt, dct) &&
1318            csrow == online_spare_bad_dramcs(pvt, dct)) {
1319
1320                for_each_chip_select(tmp_cs, dct, pvt) {
1321                        if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1322                                csrow = tmp_cs;
1323                                break;
1324                        }
1325                }
1326        }
1327        return csrow;
1328}
1329
1330/*
1331 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1332 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1333 *
1334 * Return:
1335 *      -EINVAL:  NOT FOUND
1336 *      0..csrow = Chip-Select Row
1337 */
1338static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1339{
1340        struct mem_ctl_info *mci;
1341        struct amd64_pvt *pvt;
1342        u64 cs_base, cs_mask;
1343        int cs_found = -EINVAL;
1344        int csrow;
1345
1346        mci = mcis[nid];
1347        if (!mci)
1348                return cs_found;
1349
1350        pvt = mci->pvt_info;
1351
1352        edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1353
1354        for_each_chip_select(csrow, dct, pvt) {
1355                if (!csrow_enabled(csrow, dct, pvt))
1356                        continue;
1357
1358                get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1359
1360                edac_dbg(1, "    CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1361                         csrow, cs_base, cs_mask);
1362
1363                cs_mask = ~cs_mask;
1364
1365                edac_dbg(1, "    (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1366                         (in_addr & cs_mask), (cs_base & cs_mask));
1367
1368                if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1369                        cs_found = f10_process_possible_spare(pvt, dct, csrow);
1370
1371                        edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1372                        break;
1373                }
1374        }
1375        return cs_found;
1376}
1377
1378/*
1379 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1380 * swapped with a region located at the bottom of memory so that the GPU can use
1381 * the interleaved region and thus two channels.
1382 */
1383static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1384{
1385        u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1386
1387        if (boot_cpu_data.x86 == 0x10) {
1388                /* only revC3 and revE have that feature */
1389                if (boot_cpu_data.x86_model < 4 ||
1390                    (boot_cpu_data.x86_model < 0xa &&
1391                     boot_cpu_data.x86_mask < 3))
1392                        return sys_addr;
1393        }
1394
1395        amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1396
1397        if (!(swap_reg & 0x1))
1398                return sys_addr;
1399
1400        swap_base       = (swap_reg >> 3) & 0x7f;
1401        swap_limit      = (swap_reg >> 11) & 0x7f;
1402        rgn_size        = (swap_reg >> 20) & 0x7f;
1403        tmp_addr        = sys_addr >> 27;
1404
1405        if (!(sys_addr >> 34) &&
1406            (((tmp_addr >= swap_base) &&
1407             (tmp_addr <= swap_limit)) ||
1408             (tmp_addr < rgn_size)))
1409                return sys_addr ^ (u64)swap_base << 27;
1410
1411        return sys_addr;
1412}
1413
1414/* For a given @dram_range, check if @sys_addr falls within it. */
1415static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1416                                  u64 sys_addr, int *chan_sel)
1417{
1418        int cs_found = -EINVAL;
1419        u64 chan_addr;
1420        u32 dct_sel_base;
1421        u8 channel;
1422        bool high_range = false;
1423
1424        u8 node_id    = dram_dst_node(pvt, range);
1425        u8 intlv_en   = dram_intlv_en(pvt, range);
1426        u32 intlv_sel = dram_intlv_sel(pvt, range);
1427
1428        edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1429                 range, sys_addr, get_dram_limit(pvt, range));
1430
1431        if (dhar_valid(pvt) &&
1432            dhar_base(pvt) <= sys_addr &&
1433            sys_addr < BIT_64(32)) {
1434                amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1435                            sys_addr);
1436                return -EINVAL;
1437        }
1438
1439        if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1440                return -EINVAL;
1441
1442        sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1443
1444        dct_sel_base = dct_sel_baseaddr(pvt);
1445
1446        /*
1447         * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1448         * select between DCT0 and DCT1.
1449         */
1450        if (dct_high_range_enabled(pvt) &&
1451           !dct_ganging_enabled(pvt) &&
1452           ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1453                high_range = true;
1454
1455        channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1456
1457        chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1458                                          high_range, dct_sel_base);
1459
1460        /* Remove node interleaving, see F1x120 */
1461        if (intlv_en)
1462                chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1463                            (chan_addr & 0xfff);
1464
1465        /* remove channel interleave */
1466        if (dct_interleave_enabled(pvt) &&
1467           !dct_high_range_enabled(pvt) &&
1468           !dct_ganging_enabled(pvt)) {
1469
1470                if (dct_sel_interleave_addr(pvt) != 1) {
1471                        if (dct_sel_interleave_addr(pvt) == 0x3)
1472                                /* hash 9 */
1473                                chan_addr = ((chan_addr >> 10) << 9) |
1474                                             (chan_addr & 0x1ff);
1475                        else
1476                                /* A[6] or hash 6 */
1477                                chan_addr = ((chan_addr >> 7) << 6) |
1478                                             (chan_addr & 0x3f);
1479                } else
1480                        /* A[12] */
1481                        chan_addr = ((chan_addr >> 13) << 12) |
1482                                     (chan_addr & 0xfff);
1483        }
1484
1485        edac_dbg(1, "   Normalized DCT addr: 0x%llx\n", chan_addr);
1486
1487        cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1488
1489        if (cs_found >= 0)
1490                *chan_sel = channel;
1491
1492        return cs_found;
1493}
1494
1495static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1496                                       int *chan_sel)
1497{
1498        int cs_found = -EINVAL;
1499        unsigned range;
1500
1501        for (range = 0; range < DRAM_RANGES; range++) {
1502
1503                if (!dram_rw(pvt, range))
1504                        continue;
1505
1506                if ((get_dram_base(pvt, range)  <= sys_addr) &&
1507                    (get_dram_limit(pvt, range) >= sys_addr)) {
1508
1509                        cs_found = f1x_match_to_this_node(pvt, range,
1510                                                          sys_addr, chan_sel);
1511                        if (cs_found >= 0)
1512                                break;
1513                }
1514        }
1515        return cs_found;
1516}
1517
1518/*
1519 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1520 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1521 *
1522 * The @sys_addr is usually an error address received from the hardware
1523 * (MCX_ADDR).
1524 */
1525static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1526                                     struct err_info *err)
1527{
1528        struct amd64_pvt *pvt = mci->pvt_info;
1529
1530        error_address_to_page_and_offset(sys_addr, err);
1531
1532        err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1533        if (err->csrow < 0) {
1534                err->err_code = ERR_CSROW;
1535                return;
1536        }
1537
1538        /*
1539         * We need the syndromes for channel detection only when we're
1540         * ganged. Otherwise @chan should already contain the channel at
1541         * this point.
1542         */
1543        if (dct_ganging_enabled(pvt))
1544                err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1545}
1546
1547/*
1548 * debug routine to display the memory sizes of all logical DIMMs and its
1549 * CSROWs
1550 */
1551static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1552{
1553        int dimm, size0, size1;
1554        u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1555        u32 dbam  = ctrl ? pvt->dbam1 : pvt->dbam0;
1556
1557        if (boot_cpu_data.x86 == 0xf) {
1558                /* K8 families < revF not supported yet */
1559               if (pvt->ext_model < K8_REV_F)
1560                        return;
1561               else
1562                       WARN_ON(ctrl != 0);
1563        }
1564
1565        dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1566        dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1567                                                   : pvt->csels[0].csbases;
1568
1569        edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1570                 ctrl, dbam);
1571
1572        edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1573
1574        /* Dump memory sizes for DIMM and its CSROWs */
1575        for (dimm = 0; dimm < 4; dimm++) {
1576
1577                size0 = 0;
1578                if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1579                        size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1580                                                     DBAM_DIMM(dimm, dbam));
1581
1582                size1 = 0;
1583                if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1584                        size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1585                                                     DBAM_DIMM(dimm, dbam));
1586
1587                amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1588                                dimm * 2,     size0,
1589                                dimm * 2 + 1, size1);
1590        }
1591}
1592
1593static struct amd64_family_type amd64_family_types[] = {
1594        [K8_CPUS] = {
1595                .ctl_name = "K8",
1596                .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1597                .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1598                .ops = {
1599                        .early_channel_count    = k8_early_channel_count,
1600                        .map_sysaddr_to_csrow   = k8_map_sysaddr_to_csrow,
1601                        .dbam_to_cs             = k8_dbam_to_chip_select,
1602                        .read_dct_pci_cfg       = k8_read_dct_pci_cfg,
1603                }
1604        },
1605        [F10_CPUS] = {
1606                .ctl_name = "F10h",
1607                .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1608                .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1609                .ops = {
1610                        .early_channel_count    = f1x_early_channel_count,
1611                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
1612                        .dbam_to_cs             = f10_dbam_to_chip_select,
1613                        .read_dct_pci_cfg       = f10_read_dct_pci_cfg,
1614                }
1615        },
1616        [F15_CPUS] = {
1617                .ctl_name = "F15h",
1618                .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1619                .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1620                .ops = {
1621                        .early_channel_count    = f1x_early_channel_count,
1622                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
1623                        .dbam_to_cs             = f15_dbam_to_chip_select,
1624                        .read_dct_pci_cfg       = f15_read_dct_pci_cfg,
1625                }
1626        },
1627        [F16_CPUS] = {
1628                .ctl_name = "F16h",
1629                .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
1630                .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3,
1631                .ops = {
1632                        .early_channel_count    = f1x_early_channel_count,
1633                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
1634                        .dbam_to_cs             = f16_dbam_to_chip_select,
1635                        .read_dct_pci_cfg       = f10_read_dct_pci_cfg,
1636                }
1637        },
1638};
1639
1640/*
1641 * These are tables of eigenvectors (one per line) which can be used for the
1642 * construction of the syndrome tables. The modified syndrome search algorithm
1643 * uses those to find the symbol in error and thus the DIMM.
1644 *
1645 * Algorithm courtesy of Ross LaFetra from AMD.
1646 */
1647static const u16 x4_vectors[] = {
1648        0x2f57, 0x1afe, 0x66cc, 0xdd88,
1649        0x11eb, 0x3396, 0x7f4c, 0xeac8,
1650        0x0001, 0x0002, 0x0004, 0x0008,
1651        0x1013, 0x3032, 0x4044, 0x8088,
1652        0x106b, 0x30d6, 0x70fc, 0xe0a8,
1653        0x4857, 0xc4fe, 0x13cc, 0x3288,
1654        0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1655        0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1656        0x15c1, 0x2a42, 0x89ac, 0x4758,
1657        0x2b03, 0x1602, 0x4f0c, 0xca08,
1658        0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1659        0x8ba7, 0x465e, 0x244c, 0x1cc8,
1660        0x2b87, 0x164e, 0x642c, 0xdc18,
1661        0x40b9, 0x80de, 0x1094, 0x20e8,
1662        0x27db, 0x1eb6, 0x9dac, 0x7b58,
1663        0x11c1, 0x2242, 0x84ac, 0x4c58,
1664        0x1be5, 0x2d7a, 0x5e34, 0xa718,
1665        0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1666        0x4c97, 0xc87e, 0x11fc, 0x33a8,
1667        0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1668        0x16b3, 0x3d62, 0x4f34, 0x8518,
1669        0x1e2f, 0x391a, 0x5cac, 0xf858,
1670        0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1671        0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1672        0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1673        0x4397, 0xc27e, 0x17fc, 0x3ea8,
1674        0x1617, 0x3d3e, 0x6464, 0xb8b8,
1675        0x23ff, 0x12aa, 0xab6c, 0x56d8,
1676        0x2dfb, 0x1ba6, 0x913c, 0x7328,
1677        0x185d, 0x2ca6, 0x7914, 0x9e28,
1678        0x171b, 0x3e36, 0x7d7c, 0xebe8,
1679        0x4199, 0x82ee, 0x19f4, 0x2e58,
1680        0x4807, 0xc40e, 0x130c, 0x3208,
1681        0x1905, 0x2e0a, 0x5804, 0xac08,
1682        0x213f, 0x132a, 0xadfc, 0x5ba8,
1683        0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1684};
1685
1686static const u16 x8_vectors[] = {
1687        0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1688        0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1689        0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1690        0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1691        0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1692        0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1693        0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1694        0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1695        0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1696        0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1697        0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1698        0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1699        0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1700        0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1701        0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1702        0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1703        0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1704        0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1705        0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1706};
1707
1708static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
1709                           unsigned v_dim)
1710{
1711        unsigned int i, err_sym;
1712
1713        for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1714                u16 s = syndrome;
1715                unsigned v_idx =  err_sym * v_dim;
1716                unsigned v_end = (err_sym + 1) * v_dim;
1717
1718                /* walk over all 16 bits of the syndrome */
1719                for (i = 1; i < (1U << 16); i <<= 1) {
1720
1721                        /* if bit is set in that eigenvector... */
1722                        if (v_idx < v_end && vectors[v_idx] & i) {
1723                                u16 ev_comp = vectors[v_idx++];
1724
1725                                /* ... and bit set in the modified syndrome, */
1726                                if (s & i) {
1727                                        /* remove it. */
1728                                        s ^= ev_comp;
1729
1730                                        if (!s)
1731                                                return err_sym;
1732                                }
1733
1734                        } else if (s & i)
1735                                /* can't get to zero, move to next symbol */
1736                                break;
1737                }
1738        }
1739
1740        edac_dbg(0, "syndrome(%x) not found\n", syndrome);
1741        return -1;
1742}
1743
1744static int map_err_sym_to_channel(int err_sym, int sym_size)
1745{
1746        if (sym_size == 4)
1747                switch (err_sym) {
1748                case 0x20:
1749                case 0x21:
1750                        return 0;
1751                        break;
1752                case 0x22:
1753                case 0x23:
1754                        return 1;
1755                        break;
1756                default:
1757                        return err_sym >> 4;
1758                        break;
1759                }
1760        /* x8 symbols */
1761        else
1762                switch (err_sym) {
1763                /* imaginary bits not in a DIMM */
1764                case 0x10:
1765                        WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1766                                          err_sym);
1767                        return -1;
1768                        break;
1769
1770                case 0x11:
1771                        return 0;
1772                        break;
1773                case 0x12:
1774                        return 1;
1775                        break;
1776                default:
1777                        return err_sym >> 3;
1778                        break;
1779                }
1780        return -1;
1781}
1782
1783static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1784{
1785        struct amd64_pvt *pvt = mci->pvt_info;
1786        int err_sym = -1;
1787
1788        if (pvt->ecc_sym_sz == 8)
1789                err_sym = decode_syndrome(syndrome, x8_vectors,
1790                                          ARRAY_SIZE(x8_vectors),
1791                                          pvt->ecc_sym_sz);
1792        else if (pvt->ecc_sym_sz == 4)
1793                err_sym = decode_syndrome(syndrome, x4_vectors,
1794                                          ARRAY_SIZE(x4_vectors),
1795                                          pvt->ecc_sym_sz);
1796        else {
1797                amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1798                return err_sym;
1799        }
1800
1801        return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1802}
1803
1804static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
1805                            u8 ecc_type)
1806{
1807        enum hw_event_mc_err_type err_type;
1808        const char *string;
1809
1810        if (ecc_type == 2)
1811                err_type = HW_EVENT_ERR_CORRECTED;
1812        else if (ecc_type == 1)
1813                err_type = HW_EVENT_ERR_UNCORRECTED;
1814        else {
1815                WARN(1, "Something is rotten in the state of Denmark.\n");
1816                return;
1817        }
1818
1819        switch (err->err_code) {
1820        case DECODE_OK:
1821                string = "";
1822                break;
1823        case ERR_NODE:
1824                string = "Failed to map error addr to a node";
1825                break;
1826        case ERR_CSROW:
1827                string = "Failed to map error addr to a csrow";
1828                break;
1829        case ERR_CHANNEL:
1830                string = "unknown syndrome - possible error reporting race";
1831                break;
1832        default:
1833                string = "WTF error";
1834                break;
1835        }
1836
1837        edac_mc_handle_error(err_type, mci, 1,
1838                             err->page, err->offset, err->syndrome,
1839                             err->csrow, err->channel, -1,
1840                             string, "");
1841}
1842
1843static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1844                                            struct mce *m)
1845{
1846        struct amd64_pvt *pvt = mci->pvt_info;
1847        u8 ecc_type = (m->status >> 45) & 0x3;
1848        u8 xec = XEC(m->status, 0x1f);
1849        u16 ec = EC(m->status);
1850        u64 sys_addr;
1851        struct err_info err;
1852
1853        /* Bail out early if this was an 'observed' error */
1854        if (PP(ec) == NBSL_PP_OBS)
1855                return;
1856
1857        /* Do only ECC errors */
1858        if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1859                return;
1860
1861        memset(&err, 0, sizeof(err));
1862
1863        sys_addr = get_error_address(m);
1864
1865        if (ecc_type == 2)
1866                err.syndrome = extract_syndrome(m->status);
1867
1868        pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
1869
1870        __log_bus_error(mci, &err, ecc_type);
1871}
1872
1873void amd64_decode_bus_error(int node_id, struct mce *m)
1874{
1875        __amd64_decode_bus_error(mcis[node_id], m);
1876}
1877
1878/*
1879 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1880 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1881 */
1882static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1883{
1884        /* Reserve the ADDRESS MAP Device */
1885        pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1886        if (!pvt->F1) {
1887                amd64_err("error address map device not found: "
1888                          "vendor %x device 0x%x (broken BIOS?)\n",
1889                          PCI_VENDOR_ID_AMD, f1_id);
1890                return -ENODEV;
1891        }
1892
1893        /* Reserve the MISC Device */
1894        pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1895        if (!pvt->F3) {
1896                pci_dev_put(pvt->F1);
1897                pvt->F1 = NULL;
1898
1899                amd64_err("error F3 device not found: "
1900                          "vendor %x device 0x%x (broken BIOS?)\n",
1901                          PCI_VENDOR_ID_AMD, f3_id);
1902
1903                return -ENODEV;
1904        }
1905        edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
1906        edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
1907        edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
1908
1909        return 0;
1910}
1911
1912static void free_mc_sibling_devs(struct amd64_pvt *pvt)
1913{
1914        pci_dev_put(pvt->F1);
1915        pci_dev_put(pvt->F3);
1916}
1917
1918/*
1919 * Retrieve the hardware registers of the memory controller (this includes the
1920 * 'Address Map' and 'Misc' device regs)
1921 */
1922static void read_mc_regs(struct amd64_pvt *pvt)
1923{
1924        struct cpuinfo_x86 *c = &boot_cpu_data;
1925        u64 msr_val;
1926        u32 tmp;
1927        unsigned range;
1928
1929        /*
1930         * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1931         * those are Read-As-Zero
1932         */
1933        rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
1934        edac_dbg(0, "  TOP_MEM:  0x%016llx\n", pvt->top_mem);
1935
1936        /* check first whether TOP_MEM2 is enabled */
1937        rdmsrl(MSR_K8_SYSCFG, msr_val);
1938        if (msr_val & (1U << 21)) {
1939                rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
1940                edac_dbg(0, "  TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
1941        } else
1942                edac_dbg(0, "  TOP_MEM2 disabled\n");
1943
1944        amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
1945
1946        read_dram_ctl_register(pvt);
1947
1948        for (range = 0; range < DRAM_RANGES; range++) {
1949                u8 rw;
1950
1951                /* read settings for this DRAM range */
1952                read_dram_base_limit_regs(pvt, range);
1953
1954                rw = dram_rw(pvt, range);
1955                if (!rw)
1956                        continue;
1957
1958                edac_dbg(1, "  DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1959                         range,
1960                         get_dram_base(pvt, range),
1961                         get_dram_limit(pvt, range));
1962
1963                edac_dbg(1, "   IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1964                         dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1965                         (rw & 0x1) ? "R" : "-",
1966                         (rw & 0x2) ? "W" : "-",
1967                         dram_intlv_sel(pvt, range),
1968                         dram_dst_node(pvt, range));
1969        }
1970
1971        read_dct_base_mask(pvt);
1972
1973        amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
1974        amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
1975
1976        amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
1977
1978        amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
1979        amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
1980
1981        if (!dct_ganging_enabled(pvt)) {
1982                amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
1983                amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
1984        }
1985
1986        pvt->ecc_sym_sz = 4;
1987
1988        if (c->x86 >= 0x10) {
1989                amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
1990                if (c->x86 != 0x16)
1991                        /* F16h has only DCT0 */
1992                        amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
1993
1994                /* F10h, revD and later can do x8 ECC too */
1995                if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
1996                        pvt->ecc_sym_sz = 8;
1997        }
1998        dump_misc_regs(pvt);
1999}
2000
2001/*
2002 * NOTE: CPU Revision Dependent code
2003 *
2004 * Input:
2005 *      @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2006 *      k8 private pointer to -->
2007 *                      DRAM Bank Address mapping register
2008 *                      node_id
2009 *                      DCL register where dual_channel_active is
2010 *
2011 * The DBAM register consists of 4 sets of 4 bits each definitions:
2012 *
2013 * Bits:        CSROWs
2014 * 0-3          CSROWs 0 and 1
2015 * 4-7          CSROWs 2 and 3
2016 * 8-11         CSROWs 4 and 5
2017 * 12-15        CSROWs 6 and 7
2018 *
2019 * Values range from: 0 to 15
2020 * The meaning of the values depends on CPU revision and dual-channel state,
2021 * see relevant BKDG more info.
2022 *
2023 * The memory controller provides for total of only 8 CSROWs in its current
2024 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2025 * single channel or two (2) DIMMs in dual channel mode.
2026 *
2027 * The following code logic collapses the various tables for CSROW based on CPU
2028 * revision.
2029 *
2030 * Returns:
2031 *      The number of PAGE_SIZE pages on the specified CSROW number it
2032 *      encompasses
2033 *
2034 */
2035static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2036{
2037        u32 cs_mode, nr_pages;
2038        u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2039
2040
2041        /*
2042         * The math on this doesn't look right on the surface because x/2*4 can
2043         * be simplified to x*2 but this expression makes use of the fact that
2044         * it is integral math where 1/2=0. This intermediate value becomes the
2045         * number of bits to shift the DBAM register to extract the proper CSROW
2046         * field.
2047         */
2048        cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
2049
2050        nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2051
2052        edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2053                    csrow_nr, dct,  cs_mode);
2054        edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2055
2056        return nr_pages;
2057}
2058
2059/*
2060 * Initialize the array of csrow attribute instances, based on the values
2061 * from pci config hardware registers.
2062 */
2063static int init_csrows(struct mem_ctl_info *mci)
2064{
2065        struct amd64_pvt *pvt = mci->pvt_info;
2066        struct csrow_info *csrow;
2067        struct dimm_info *dimm;
2068        enum edac_type edac_mode;
2069        enum mem_type mtype;
2070        int i, j, empty = 1;
2071        int nr_pages = 0;
2072        u32 val;
2073
2074        amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2075
2076        pvt->nbcfg = val;
2077
2078        edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2079                 pvt->mc_node_id, val,
2080                 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2081
2082        /*
2083         * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2084         */
2085        for_each_chip_select(i, 0, pvt) {
2086                bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2087                bool row_dct1 = false;
2088
2089                if (boot_cpu_data.x86 != 0xf)
2090                        row_dct1 = !!csrow_enabled(i, 1, pvt);
2091
2092                if (!row_dct0 && !row_dct1)
2093                        continue;
2094
2095                csrow = mci->csrows[i];
2096                empty = 0;
2097
2098                edac_dbg(1, "MC node: %d, csrow: %d\n",
2099                            pvt->mc_node_id, i);
2100
2101                if (row_dct0) {
2102                        nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2103                        csrow->channels[0]->dimm->nr_pages = nr_pages;
2104                }
2105
2106                /* K8 has only one DCT */
2107                if (boot_cpu_data.x86 != 0xf && row_dct1) {
2108                        int row_dct1_pages = amd64_csrow_nr_pages(pvt, 1, i);
2109
2110                        csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2111                        nr_pages += row_dct1_pages;
2112                }
2113
2114                mtype = amd64_determine_memory_type(pvt, i);
2115
2116                edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2117
2118                /*
2119                 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2120                 */
2121                if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2122                        edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2123                                    EDAC_S4ECD4ED : EDAC_SECDED;
2124                else
2125                        edac_mode = EDAC_NONE;
2126
2127                for (j = 0; j < pvt->channel_count; j++) {
2128                        dimm = csrow->channels[j]->dimm;
2129                        dimm->mtype = mtype;
2130                        dimm->edac_mode = edac_mode;
2131                }
2132        }
2133
2134        return empty;
2135}
2136
2137/* get all cores on this DCT */
2138static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2139{
2140        int cpu;
2141
2142        for_each_online_cpu(cpu)
2143                if (amd_get_nb_id(cpu) == nid)
2144                        cpumask_set_cpu(cpu, mask);
2145}
2146
2147/* check MCG_CTL on all the cpus on this node */
2148static bool amd64_nb_mce_bank_enabled_on_node(u16 nid)
2149{
2150        cpumask_var_t mask;
2151        int cpu, nbe;
2152        bool ret = false;
2153
2154        if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2155                amd64_warn("%s: Error allocating mask\n", __func__);
2156                return false;
2157        }
2158
2159        get_cpus_on_this_dct_cpumask(mask, nid);
2160
2161        rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2162
2163        for_each_cpu(cpu, mask) {
2164                struct msr *reg = per_cpu_ptr(msrs, cpu);
2165                nbe = reg->l & MSR_MCGCTL_NBE;
2166
2167                edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2168                         cpu, reg->q,
2169                         (nbe ? "enabled" : "disabled"));
2170
2171                if (!nbe)
2172                        goto out;
2173        }
2174        ret = true;
2175
2176out:
2177        free_cpumask_var(mask);
2178        return ret;
2179}
2180
2181static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2182{
2183        cpumask_var_t cmask;
2184        int cpu;
2185
2186        if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2187                amd64_warn("%s: error allocating mask\n", __func__);
2188                return false;
2189        }
2190
2191        get_cpus_on_this_dct_cpumask(cmask, nid);
2192
2193        rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2194
2195        for_each_cpu(cpu, cmask) {
2196
2197                struct msr *reg = per_cpu_ptr(msrs, cpu);
2198
2199                if (on) {
2200                        if (reg->l & MSR_MCGCTL_NBE)
2201                                s->flags.nb_mce_enable = 1;
2202
2203                        reg->l |= MSR_MCGCTL_NBE;
2204                } else {
2205                        /*
2206                         * Turn off NB MCE reporting only when it was off before
2207                         */
2208                        if (!s->flags.nb_mce_enable)
2209                                reg->l &= ~MSR_MCGCTL_NBE;
2210                }
2211        }
2212        wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2213
2214        free_cpumask_var(cmask);
2215
2216        return 0;
2217}
2218
2219static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2220                                       struct pci_dev *F3)
2221{
2222        bool ret = true;
2223        u32 value, mask = 0x3;          /* UECC/CECC enable */
2224
2225        if (toggle_ecc_err_reporting(s, nid, ON)) {
2226                amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2227                return false;
2228        }
2229
2230        amd64_read_pci_cfg(F3, NBCTL, &value);
2231
2232        s->old_nbctl   = value & mask;
2233        s->nbctl_valid = true;
2234
2235        value |= mask;
2236        amd64_write_pci_cfg(F3, NBCTL, value);
2237
2238        amd64_read_pci_cfg(F3, NBCFG, &value);
2239
2240        edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2241                 nid, value, !!(value & NBCFG_ECC_ENABLE));
2242
2243        if (!(value & NBCFG_ECC_ENABLE)) {
2244                amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2245
2246                s->flags.nb_ecc_prev = 0;
2247
2248                /* Attempt to turn on DRAM ECC Enable */
2249                value |= NBCFG_ECC_ENABLE;
2250                amd64_write_pci_cfg(F3, NBCFG, value);
2251
2252                amd64_read_pci_cfg(F3, NBCFG, &value);
2253
2254                if (!(value & NBCFG_ECC_ENABLE)) {
2255                        amd64_warn("Hardware rejected DRAM ECC enable,"
2256                                   "check memory DIMM configuration.\n");
2257                        ret = false;
2258                } else {
2259                        amd64_info("Hardware accepted DRAM ECC Enable\n");
2260                }
2261        } else {
2262                s->flags.nb_ecc_prev = 1;
2263        }
2264
2265        edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2266                 nid, value, !!(value & NBCFG_ECC_ENABLE));
2267
2268        return ret;
2269}
2270
2271static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2272                                        struct pci_dev *F3)
2273{
2274        u32 value, mask = 0x3;          /* UECC/CECC enable */
2275
2276
2277        if (!s->nbctl_valid)
2278                return;
2279
2280        amd64_read_pci_cfg(F3, NBCTL, &value);
2281        value &= ~mask;
2282        value |= s->old_nbctl;
2283
2284        amd64_write_pci_cfg(F3, NBCTL, value);
2285
2286        /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2287        if (!s->flags.nb_ecc_prev) {
2288                amd64_read_pci_cfg(F3, NBCFG, &value);
2289                value &= ~NBCFG_ECC_ENABLE;
2290                amd64_write_pci_cfg(F3, NBCFG, value);
2291        }
2292
2293        /* restore the NB Enable MCGCTL bit */
2294        if (toggle_ecc_err_reporting(s, nid, OFF))
2295                amd64_warn("Error restoring NB MCGCTL settings!\n");
2296}
2297
2298/*
2299 * EDAC requires that the BIOS have ECC enabled before
2300 * taking over the processing of ECC errors. A command line
2301 * option allows to force-enable hardware ECC later in
2302 * enable_ecc_error_reporting().
2303 */
2304static const char *ecc_msg =
2305        "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2306        " Either enable ECC checking or force module loading by setting "
2307        "'ecc_enable_override'.\n"
2308        " (Note that use of the override may cause unknown side effects.)\n";
2309
2310static bool ecc_enabled(struct pci_dev *F3, u16 nid)
2311{
2312        u32 value;
2313        u8 ecc_en = 0;
2314        bool nb_mce_en = false;
2315
2316        amd64_read_pci_cfg(F3, NBCFG, &value);
2317
2318        ecc_en = !!(value & NBCFG_ECC_ENABLE);
2319        amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2320
2321        nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2322        if (!nb_mce_en)
2323                amd64_notice("NB MCE bank disabled, set MSR "
2324                             "0x%08x[4] on node %d to enable.\n",
2325                             MSR_IA32_MCG_CTL, nid);
2326
2327        if (!ecc_en || !nb_mce_en) {
2328                amd64_notice("%s", ecc_msg);
2329                return false;
2330        }
2331        return true;
2332}
2333
2334static int set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2335{
2336        int rc;
2337
2338        rc = amd64_create_sysfs_dbg_files(mci);
2339        if (rc < 0)
2340                return rc;
2341
2342        if (boot_cpu_data.x86 >= 0x10) {
2343                rc = amd64_create_sysfs_inject_files(mci);
2344                if (rc < 0)
2345                        return rc;
2346        }
2347
2348        return 0;
2349}
2350
2351static void del_mc_sysfs_attrs(struct mem_ctl_info *mci)
2352{
2353        amd64_remove_sysfs_dbg_files(mci);
2354
2355        if (boot_cpu_data.x86 >= 0x10)
2356                amd64_remove_sysfs_inject_files(mci);
2357}
2358
2359static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2360                                 struct amd64_family_type *fam)
2361{
2362        struct amd64_pvt *pvt = mci->pvt_info;
2363
2364        mci->mtype_cap          = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2365        mci->edac_ctl_cap       = EDAC_FLAG_NONE;
2366
2367        if (pvt->nbcap & NBCAP_SECDED)
2368                mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2369
2370        if (pvt->nbcap & NBCAP_CHIPKILL)
2371                mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2372
2373        mci->edac_cap           = amd64_determine_edac_cap(pvt);
2374        mci->mod_name           = EDAC_MOD_STR;
2375        mci->mod_ver            = EDAC_AMD64_VERSION;
2376        mci->ctl_name           = fam->ctl_name;
2377        mci->dev_name           = pci_name(pvt->F2);
2378        mci->ctl_page_to_phys   = NULL;
2379
2380        /* memory scrubber interface */
2381        mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2382        mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2383}
2384
2385/*
2386 * returns a pointer to the family descriptor on success, NULL otherwise.
2387 */
2388static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2389{
2390        u8 fam = boot_cpu_data.x86;
2391        struct amd64_family_type *fam_type = NULL;
2392
2393        switch (fam) {
2394        case 0xf:
2395                fam_type                = &amd64_family_types[K8_CPUS];
2396                pvt->ops                = &amd64_family_types[K8_CPUS].ops;
2397                break;
2398
2399        case 0x10:
2400                fam_type                = &amd64_family_types[F10_CPUS];
2401                pvt->ops                = &amd64_family_types[F10_CPUS].ops;
2402                break;
2403
2404        case 0x15:
2405                fam_type                = &amd64_family_types[F15_CPUS];
2406                pvt->ops                = &amd64_family_types[F15_CPUS].ops;
2407                break;
2408
2409        case 0x16:
2410                fam_type                = &amd64_family_types[F16_CPUS];
2411                pvt->ops                = &amd64_family_types[F16_CPUS].ops;
2412                break;
2413
2414        default:
2415                amd64_err("Unsupported family!\n");
2416                return NULL;
2417        }
2418
2419        pvt->ext_model = boot_cpu_data.x86_model >> 4;
2420
2421        amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2422                     (fam == 0xf ?
2423                                (pvt->ext_model >= K8_REV_F  ? "revF or later "
2424                                                             : "revE or earlier ")
2425                                 : ""), pvt->mc_node_id);
2426        return fam_type;
2427}
2428
2429static int amd64_init_one_instance(struct pci_dev *F2)
2430{
2431        struct amd64_pvt *pvt = NULL;
2432        struct amd64_family_type *fam_type = NULL;
2433        struct mem_ctl_info *mci = NULL;
2434        struct edac_mc_layer layers[2];
2435        int err = 0, ret;
2436        u16 nid = amd_get_node_id(F2);
2437
2438        ret = -ENOMEM;
2439        pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2440        if (!pvt)
2441                goto err_ret;
2442
2443        pvt->mc_node_id = nid;
2444        pvt->F2 = F2;
2445
2446        ret = -EINVAL;
2447        fam_type = amd64_per_family_init(pvt);
2448        if (!fam_type)
2449                goto err_free;
2450
2451        ret = -ENODEV;
2452        err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2453        if (err)
2454                goto err_free;
2455
2456        read_mc_regs(pvt);
2457
2458        /*
2459         * We need to determine how many memory channels there are. Then use
2460         * that information for calculating the size of the dynamic instance
2461         * tables in the 'mci' structure.
2462         */
2463        ret = -EINVAL;
2464        pvt->channel_count = pvt->ops->early_channel_count(pvt);
2465        if (pvt->channel_count < 0)
2466                goto err_siblings;
2467
2468        ret = -ENOMEM;
2469        layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2470        layers[0].size = pvt->csels[0].b_cnt;
2471        layers[0].is_virt_csrow = true;
2472        layers[1].type = EDAC_MC_LAYER_CHANNEL;
2473        layers[1].size = pvt->channel_count;
2474        layers[1].is_virt_csrow = false;
2475        mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
2476        if (!mci)
2477                goto err_siblings;
2478
2479        mci->pvt_info = pvt;
2480        mci->pdev = &pvt->F2->dev;
2481
2482        setup_mci_misc_attrs(mci, fam_type);
2483
2484        if (init_csrows(mci))
2485                mci->edac_cap = EDAC_FLAG_NONE;
2486
2487        ret = -ENODEV;
2488        if (edac_mc_add_mc(mci)) {
2489                edac_dbg(1, "failed edac_mc_add_mc()\n");
2490                goto err_add_mc;
2491        }
2492        if (set_mc_sysfs_attrs(mci)) {
2493                edac_dbg(1, "failed edac_mc_add_mc()\n");
2494                goto err_add_sysfs;
2495        }
2496
2497        /* register stuff with EDAC MCE */
2498        if (report_gart_errors)
2499                amd_report_gart_errors(true);
2500
2501        amd_register_ecc_decoder(amd64_decode_bus_error);
2502
2503        mcis[nid] = mci;
2504
2505        atomic_inc(&drv_instances);
2506
2507        return 0;
2508
2509err_add_sysfs:
2510        edac_mc_del_mc(mci->pdev);
2511err_add_mc:
2512        edac_mc_free(mci);
2513
2514err_siblings:
2515        free_mc_sibling_devs(pvt);
2516
2517err_free:
2518        kfree(pvt);
2519
2520err_ret:
2521        return ret;
2522}
2523
2524static int amd64_probe_one_instance(struct pci_dev *pdev,
2525                                    const struct pci_device_id *mc_type)
2526{
2527        u16 nid = amd_get_node_id(pdev);
2528        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2529        struct ecc_settings *s;
2530        int ret = 0;
2531
2532        ret = pci_enable_device(pdev);
2533        if (ret < 0) {
2534                edac_dbg(0, "ret=%d\n", ret);
2535                return -EIO;
2536        }
2537
2538        ret = -ENOMEM;
2539        s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2540        if (!s)
2541                goto err_out;
2542
2543        ecc_stngs[nid] = s;
2544
2545        if (!ecc_enabled(F3, nid)) {
2546                ret = -ENODEV;
2547
2548                if (!ecc_enable_override)
2549                        goto err_enable;
2550
2551                amd64_warn("Forcing ECC on!\n");
2552
2553                if (!enable_ecc_error_reporting(s, nid, F3))
2554                        goto err_enable;
2555        }
2556
2557        ret = amd64_init_one_instance(pdev);
2558        if (ret < 0) {
2559                amd64_err("Error probing instance: %d\n", nid);
2560                restore_ecc_error_reporting(s, nid, F3);
2561        }
2562
2563        return ret;
2564
2565err_enable:
2566        kfree(s);
2567        ecc_stngs[nid] = NULL;
2568
2569err_out:
2570        return ret;
2571}
2572
2573static void amd64_remove_one_instance(struct pci_dev *pdev)
2574{
2575        struct mem_ctl_info *mci;
2576        struct amd64_pvt *pvt;
2577        u16 nid = amd_get_node_id(pdev);
2578        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2579        struct ecc_settings *s = ecc_stngs[nid];
2580
2581        mci = find_mci_by_dev(&pdev->dev);
2582        del_mc_sysfs_attrs(mci);
2583        /* Remove from EDAC CORE tracking list */
2584        mci = edac_mc_del_mc(&pdev->dev);
2585        if (!mci)
2586                return;
2587
2588        pvt = mci->pvt_info;
2589
2590        restore_ecc_error_reporting(s, nid, F3);
2591
2592        free_mc_sibling_devs(pvt);
2593
2594        /* unregister from EDAC MCE */
2595        amd_report_gart_errors(false);
2596        amd_unregister_ecc_decoder(amd64_decode_bus_error);
2597
2598        kfree(ecc_stngs[nid]);
2599        ecc_stngs[nid] = NULL;
2600
2601        /* Free the EDAC CORE resources */
2602        mci->pvt_info = NULL;
2603        mcis[nid] = NULL;
2604
2605        kfree(pvt);
2606        edac_mc_free(mci);
2607}
2608
2609/*
2610 * This table is part of the interface for loading drivers for PCI devices. The
2611 * PCI core identifies what devices are on a system during boot, and then
2612 * inquiry this table to see if this driver is for a given device found.
2613 */
2614static DEFINE_PCI_DEVICE_TABLE(amd64_pci_table) = {
2615        {
2616                .vendor         = PCI_VENDOR_ID_AMD,
2617                .device         = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2618                .subvendor      = PCI_ANY_ID,
2619                .subdevice      = PCI_ANY_ID,
2620                .class          = 0,
2621                .class_mask     = 0,
2622        },
2623        {
2624                .vendor         = PCI_VENDOR_ID_AMD,
2625                .device         = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2626                .subvendor      = PCI_ANY_ID,
2627                .subdevice      = PCI_ANY_ID,
2628                .class          = 0,
2629                .class_mask     = 0,
2630        },
2631        {
2632                .vendor         = PCI_VENDOR_ID_AMD,
2633                .device         = PCI_DEVICE_ID_AMD_15H_NB_F2,
2634                .subvendor      = PCI_ANY_ID,
2635                .subdevice      = PCI_ANY_ID,
2636                .class          = 0,
2637                .class_mask     = 0,
2638        },
2639        {
2640                .vendor         = PCI_VENDOR_ID_AMD,
2641                .device         = PCI_DEVICE_ID_AMD_16H_NB_F2,
2642                .subvendor      = PCI_ANY_ID,
2643                .subdevice      = PCI_ANY_ID,
2644                .class          = 0,
2645                .class_mask     = 0,
2646        },
2647
2648        {0, }
2649};
2650MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2651
2652static struct pci_driver amd64_pci_driver = {
2653        .name           = EDAC_MOD_STR,
2654        .probe          = amd64_probe_one_instance,
2655        .remove         = amd64_remove_one_instance,
2656        .id_table       = amd64_pci_table,
2657};
2658
2659static void setup_pci_device(void)
2660{
2661        struct mem_ctl_info *mci;
2662        struct amd64_pvt *pvt;
2663
2664        if (amd64_ctl_pci)
2665                return;
2666
2667        mci = mcis[0];
2668        if (mci) {
2669
2670                pvt = mci->pvt_info;
2671                amd64_ctl_pci =
2672                        edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2673
2674                if (!amd64_ctl_pci) {
2675                        pr_warning("%s(): Unable to create PCI control\n",
2676                                   __func__);
2677
2678                        pr_warning("%s(): PCI error report via EDAC not set\n",
2679                                   __func__);
2680                        }
2681        }
2682}
2683
2684static int __init amd64_edac_init(void)
2685{
2686        int err = -ENODEV;
2687
2688        printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2689
2690        opstate_init();
2691
2692        if (amd_cache_northbridges() < 0)
2693                goto err_ret;
2694
2695        err = -ENOMEM;
2696        mcis      = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2697        ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2698        if (!(mcis && ecc_stngs))
2699                goto err_free;
2700
2701        msrs = msrs_alloc();
2702        if (!msrs)
2703                goto err_free;
2704
2705        err = pci_register_driver(&amd64_pci_driver);
2706        if (err)
2707                goto err_pci;
2708
2709        err = -ENODEV;
2710        if (!atomic_read(&drv_instances))
2711                goto err_no_instances;
2712
2713        setup_pci_device();
2714        return 0;
2715
2716err_no_instances:
2717        pci_unregister_driver(&amd64_pci_driver);
2718
2719err_pci:
2720        msrs_free(msrs);
2721        msrs = NULL;
2722
2723err_free:
2724        kfree(mcis);
2725        mcis = NULL;
2726
2727        kfree(ecc_stngs);
2728        ecc_stngs = NULL;
2729
2730err_ret:
2731        return err;
2732}
2733
2734static void __exit amd64_edac_exit(void)
2735{
2736        if (amd64_ctl_pci)
2737                edac_pci_release_generic_ctl(amd64_ctl_pci);
2738
2739        pci_unregister_driver(&amd64_pci_driver);
2740
2741        kfree(ecc_stngs);
2742        ecc_stngs = NULL;
2743
2744        kfree(mcis);
2745        mcis = NULL;
2746
2747        msrs_free(msrs);
2748        msrs = NULL;
2749}
2750
2751module_init(amd64_edac_init);
2752module_exit(amd64_edac_exit);
2753
2754MODULE_LICENSE("GPL");
2755MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2756                "Dave Peterson, Thayne Harbaugh");
2757MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2758                EDAC_AMD64_VERSION);
2759
2760module_param(edac_op_state, int, 0444);
2761MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
2762
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.