linux/drivers/edac/amd64_edac.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2#include "amd64_edac.h"
   3#include <asm/amd_nb.h>
   4
   5static struct edac_pci_ctl_info *pci_ctl;
   6
   7/*
   8 * Set by command line parameter. If BIOS has enabled the ECC, this override is
   9 * cleared to prevent re-enabling the hardware by this driver.
  10 */
  11static int ecc_enable_override;
  12module_param(ecc_enable_override, int, 0644);
  13
  14static struct msr __percpu *msrs;
  15
  16static struct amd64_family_type *fam_type;
  17
  18/* Per-node stuff */
  19static struct ecc_settings **ecc_stngs;
  20
  21/* Device for the PCI component */
  22static struct device *pci_ctl_dev;
  23
  24/*
  25 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
  26 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
  27 * or higher value'.
  28 *
  29 *FIXME: Produce a better mapping/linearisation.
  30 */
  31static const struct scrubrate {
  32       u32 scrubval;           /* bit pattern for scrub rate */
  33       u32 bandwidth;          /* bandwidth consumed (bytes/sec) */
  34} scrubrates[] = {
  35        { 0x01, 1600000000UL},
  36        { 0x02, 800000000UL},
  37        { 0x03, 400000000UL},
  38        { 0x04, 200000000UL},
  39        { 0x05, 100000000UL},
  40        { 0x06, 50000000UL},
  41        { 0x07, 25000000UL},
  42        { 0x08, 12284069UL},
  43        { 0x09, 6274509UL},
  44        { 0x0A, 3121951UL},
  45        { 0x0B, 1560975UL},
  46        { 0x0C, 781440UL},
  47        { 0x0D, 390720UL},
  48        { 0x0E, 195300UL},
  49        { 0x0F, 97650UL},
  50        { 0x10, 48854UL},
  51        { 0x11, 24427UL},
  52        { 0x12, 12213UL},
  53        { 0x13, 6101UL},
  54        { 0x14, 3051UL},
  55        { 0x15, 1523UL},
  56        { 0x16, 761UL},
  57        { 0x00, 0UL},        /* scrubbing off */
  58};
  59
  60int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
  61                               u32 *val, const char *func)
  62{
  63        int err = 0;
  64
  65        err = pci_read_config_dword(pdev, offset, val);
  66        if (err)
  67                amd64_warn("%s: error reading F%dx%03x.\n",
  68                           func, PCI_FUNC(pdev->devfn), offset);
  69
  70        return err;
  71}
  72
  73int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
  74                                u32 val, const char *func)
  75{
  76        int err = 0;
  77
  78        err = pci_write_config_dword(pdev, offset, val);
  79        if (err)
  80                amd64_warn("%s: error writing to F%dx%03x.\n",
  81                           func, PCI_FUNC(pdev->devfn), offset);
  82
  83        return err;
  84}
  85
  86/*
  87 * Select DCT to which PCI cfg accesses are routed
  88 */
  89static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
  90{
  91        u32 reg = 0;
  92
  93        amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
  94        reg &= (pvt->model == 0x30) ? ~3 : ~1;
  95        reg |= dct;
  96        amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
  97}
  98
  99/*
 100 *
 101 * Depending on the family, F2 DCT reads need special handling:
 102 *
 103 * K8: has a single DCT only and no address offsets >= 0x100
 104 *
 105 * F10h: each DCT has its own set of regs
 106 *      DCT0 -> F2x040..
 107 *      DCT1 -> F2x140..
 108 *
 109 * F16h: has only 1 DCT
 110 *
 111 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
 112 */
 113static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
 114                                         int offset, u32 *val)
 115{
 116        switch (pvt->fam) {
 117        case 0xf:
 118                if (dct || offset >= 0x100)
 119                        return -EINVAL;
 120                break;
 121
 122        case 0x10:
 123                if (dct) {
 124                        /*
 125                         * Note: If ganging is enabled, barring the regs
 126                         * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
 127                         * return 0. (cf. Section 2.8.1 F10h BKDG)
 128                         */
 129                        if (dct_ganging_enabled(pvt))
 130                                return 0;
 131
 132                        offset += 0x100;
 133                }
 134                break;
 135
 136        case 0x15:
 137                /*
 138                 * F15h: F2x1xx addresses do not map explicitly to DCT1.
 139                 * We should select which DCT we access using F1x10C[DctCfgSel]
 140                 */
 141                dct = (dct && pvt->model == 0x30) ? 3 : dct;
 142                f15h_select_dct(pvt, dct);
 143                break;
 144
 145        case 0x16:
 146                if (dct)
 147                        return -EINVAL;
 148                break;
 149
 150        default:
 151                break;
 152        }
 153        return amd64_read_pci_cfg(pvt->F2, offset, val);
 154}
 155
 156/*
 157 * Memory scrubber control interface. For K8, memory scrubbing is handled by
 158 * hardware and can involve L2 cache, dcache as well as the main memory. With
 159 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
 160 * functionality.
 161 *
 162 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
 163 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
 164 * bytes/sec for the setting.
 165 *
 166 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
 167 * other archs, we might not have access to the caches directly.
 168 */
 169
 170static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
 171{
 172        /*
 173         * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
 174         * are shifted down by 0x5, so scrubval 0x5 is written to the register
 175         * as 0x0, scrubval 0x6 as 0x1, etc.
 176         */
 177        if (scrubval >= 0x5 && scrubval <= 0x14) {
 178                scrubval -= 0x5;
 179                pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
 180                pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
 181        } else {
 182                pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
 183        }
 184}
 185/*
 186 * Scan the scrub rate mapping table for a close or matching bandwidth value to
 187 * issue. If requested is too big, then use last maximum value found.
 188 */
 189static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
 190{
 191        u32 scrubval;
 192        int i;
 193
 194        /*
 195         * map the configured rate (new_bw) to a value specific to the AMD64
 196         * memory controller and apply to register. Search for the first
 197         * bandwidth entry that is greater or equal than the setting requested
 198         * and program that. If at last entry, turn off DRAM scrubbing.
 199         *
 200         * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
 201         * by falling back to the last element in scrubrates[].
 202         */
 203        for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
 204                /*
 205                 * skip scrub rates which aren't recommended
 206                 * (see F10 BKDG, F3x58)
 207                 */
 208                if (scrubrates[i].scrubval < min_rate)
 209                        continue;
 210
 211                if (scrubrates[i].bandwidth <= new_bw)
 212                        break;
 213        }
 214
 215        scrubval = scrubrates[i].scrubval;
 216
 217        if (pvt->umc) {
 218                __f17h_set_scrubval(pvt, scrubval);
 219        } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
 220                f15h_select_dct(pvt, 0);
 221                pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
 222                f15h_select_dct(pvt, 1);
 223                pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
 224        } else {
 225                pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
 226        }
 227
 228        if (scrubval)
 229                return scrubrates[i].bandwidth;
 230
 231        return 0;
 232}
 233
 234static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
 235{
 236        struct amd64_pvt *pvt = mci->pvt_info;
 237        u32 min_scrubrate = 0x5;
 238
 239        if (pvt->fam == 0xf)
 240                min_scrubrate = 0x0;
 241
 242        if (pvt->fam == 0x15) {
 243                /* Erratum #505 */
 244                if (pvt->model < 0x10)
 245                        f15h_select_dct(pvt, 0);
 246
 247                if (pvt->model == 0x60)
 248                        min_scrubrate = 0x6;
 249        }
 250        return __set_scrub_rate(pvt, bw, min_scrubrate);
 251}
 252
 253static int get_scrub_rate(struct mem_ctl_info *mci)
 254{
 255        struct amd64_pvt *pvt = mci->pvt_info;
 256        int i, retval = -EINVAL;
 257        u32 scrubval = 0;
 258
 259        if (pvt->umc) {
 260                amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
 261                if (scrubval & BIT(0)) {
 262                        amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
 263                        scrubval &= 0xF;
 264                        scrubval += 0x5;
 265                } else {
 266                        scrubval = 0;
 267                }
 268        } else if (pvt->fam == 0x15) {
 269                /* Erratum #505 */
 270                if (pvt->model < 0x10)
 271                        f15h_select_dct(pvt, 0);
 272
 273                if (pvt->model == 0x60)
 274                        amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
 275                else
 276                        amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
 277        } else {
 278                amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
 279        }
 280
 281        scrubval = scrubval & 0x001F;
 282
 283        for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
 284                if (scrubrates[i].scrubval == scrubval) {
 285                        retval = scrubrates[i].bandwidth;
 286                        break;
 287                }
 288        }
 289        return retval;
 290}
 291
 292/*
 293 * returns true if the SysAddr given by sys_addr matches the
 294 * DRAM base/limit associated with node_id
 295 */
 296static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
 297{
 298        u64 addr;
 299
 300        /* The K8 treats this as a 40-bit value.  However, bits 63-40 will be
 301         * all ones if the most significant implemented address bit is 1.
 302         * Here we discard bits 63-40.  See section 3.4.2 of AMD publication
 303         * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
 304         * Application Programming.
 305         */
 306        addr = sys_addr & 0x000000ffffffffffull;
 307
 308        return ((addr >= get_dram_base(pvt, nid)) &&
 309                (addr <= get_dram_limit(pvt, nid)));
 310}
 311
 312/*
 313 * Attempt to map a SysAddr to a node. On success, return a pointer to the
 314 * mem_ctl_info structure for the node that the SysAddr maps to.
 315 *
 316 * On failure, return NULL.
 317 */
 318static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
 319                                                u64 sys_addr)
 320{
 321        struct amd64_pvt *pvt;
 322        u8 node_id;
 323        u32 intlv_en, bits;
 324
 325        /*
 326         * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
 327         * 3.4.4.2) registers to map the SysAddr to a node ID.
 328         */
 329        pvt = mci->pvt_info;
 330
 331        /*
 332         * The value of this field should be the same for all DRAM Base
 333         * registers.  Therefore we arbitrarily choose to read it from the
 334         * register for node 0.
 335         */
 336        intlv_en = dram_intlv_en(pvt, 0);
 337
 338        if (intlv_en == 0) {
 339                for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
 340                        if (base_limit_match(pvt, sys_addr, node_id))
 341                                goto found;
 342                }
 343                goto err_no_match;
 344        }
 345
 346        if (unlikely((intlv_en != 0x01) &&
 347                     (intlv_en != 0x03) &&
 348                     (intlv_en != 0x07))) {
 349                amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
 350                return NULL;
 351        }
 352
 353        bits = (((u32) sys_addr) >> 12) & intlv_en;
 354
 355        for (node_id = 0; ; ) {
 356                if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
 357                        break;  /* intlv_sel field matches */
 358
 359                if (++node_id >= DRAM_RANGES)
 360                        goto err_no_match;
 361        }
 362
 363        /* sanity test for sys_addr */
 364        if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
 365                amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
 366                           "range for node %d with node interleaving enabled.\n",
 367                           __func__, sys_addr, node_id);
 368                return NULL;
 369        }
 370
 371found:
 372        return edac_mc_find((int)node_id);
 373
 374err_no_match:
 375        edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
 376                 (unsigned long)sys_addr);
 377
 378        return NULL;
 379}
 380
 381/*
 382 * compute the CS base address of the @csrow on the DRAM controller @dct.
 383 * For details see F2x[5C:40] in the processor's BKDG
 384 */
 385static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
 386                                 u64 *base, u64 *mask)
 387{
 388        u64 csbase, csmask, base_bits, mask_bits;
 389        u8 addr_shift;
 390
 391        if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
 392                csbase          = pvt->csels[dct].csbases[csrow];
 393                csmask          = pvt->csels[dct].csmasks[csrow];
 394                base_bits       = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
 395                mask_bits       = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
 396                addr_shift      = 4;
 397
 398        /*
 399         * F16h and F15h, models 30h and later need two addr_shift values:
 400         * 8 for high and 6 for low (cf. F16h BKDG).
 401         */
 402        } else if (pvt->fam == 0x16 ||
 403                  (pvt->fam == 0x15 && pvt->model >= 0x30)) {
 404                csbase          = pvt->csels[dct].csbases[csrow];
 405                csmask          = pvt->csels[dct].csmasks[csrow >> 1];
 406
 407                *base  = (csbase & GENMASK_ULL(15,  5)) << 6;
 408                *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
 409
 410                *mask = ~0ULL;
 411                /* poke holes for the csmask */
 412                *mask &= ~((GENMASK_ULL(15, 5)  << 6) |
 413                           (GENMASK_ULL(30, 19) << 8));
 414
 415                *mask |= (csmask & GENMASK_ULL(15, 5))  << 6;
 416                *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
 417
 418                return;
 419        } else {
 420                csbase          = pvt->csels[dct].csbases[csrow];
 421                csmask          = pvt->csels[dct].csmasks[csrow >> 1];
 422                addr_shift      = 8;
 423
 424                if (pvt->fam == 0x15)
 425                        base_bits = mask_bits =
 426                                GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
 427                else
 428                        base_bits = mask_bits =
 429                                GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
 430        }
 431
 432        *base  = (csbase & base_bits) << addr_shift;
 433
 434        *mask  = ~0ULL;
 435        /* poke holes for the csmask */
 436        *mask &= ~(mask_bits << addr_shift);
 437        /* OR them in */
 438        *mask |= (csmask & mask_bits) << addr_shift;
 439}
 440
 441#define for_each_chip_select(i, dct, pvt) \
 442        for (i = 0; i < pvt->csels[dct].b_cnt; i++)
 443
 444#define chip_select_base(i, dct, pvt) \
 445        pvt->csels[dct].csbases[i]
 446
 447#define for_each_chip_select_mask(i, dct, pvt) \
 448        for (i = 0; i < pvt->csels[dct].m_cnt; i++)
 449
 450#define for_each_umc(i) \
 451        for (i = 0; i < fam_type->max_mcs; i++)
 452
 453/*
 454 * @input_addr is an InputAddr associated with the node given by mci. Return the
 455 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
 456 */
 457static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
 458{
 459        struct amd64_pvt *pvt;
 460        int csrow;
 461        u64 base, mask;
 462
 463        pvt = mci->pvt_info;
 464
 465        for_each_chip_select(csrow, 0, pvt) {
 466                if (!csrow_enabled(csrow, 0, pvt))
 467                        continue;
 468
 469                get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
 470
 471                mask = ~mask;
 472
 473                if ((input_addr & mask) == (base & mask)) {
 474                        edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
 475                                 (unsigned long)input_addr, csrow,
 476                                 pvt->mc_node_id);
 477
 478                        return csrow;
 479                }
 480        }
 481        edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
 482                 (unsigned long)input_addr, pvt->mc_node_id);
 483
 484        return -1;
 485}
 486
 487/*
 488 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
 489 * for the node represented by mci. Info is passed back in *hole_base,
 490 * *hole_offset, and *hole_size.  Function returns 0 if info is valid or 1 if
 491 * info is invalid. Info may be invalid for either of the following reasons:
 492 *
 493 * - The revision of the node is not E or greater.  In this case, the DRAM Hole
 494 *   Address Register does not exist.
 495 *
 496 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
 497 *   indicating that its contents are not valid.
 498 *
 499 * The values passed back in *hole_base, *hole_offset, and *hole_size are
 500 * complete 32-bit values despite the fact that the bitfields in the DHAR
 501 * only represent bits 31-24 of the base and offset values.
 502 */
 503static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
 504                              u64 *hole_offset, u64 *hole_size)
 505{
 506        struct amd64_pvt *pvt = mci->pvt_info;
 507
 508        /* only revE and later have the DRAM Hole Address Register */
 509        if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
 510                edac_dbg(1, "  revision %d for node %d does not support DHAR\n",
 511                         pvt->ext_model, pvt->mc_node_id);
 512                return 1;
 513        }
 514
 515        /* valid for Fam10h and above */
 516        if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
 517                edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this system\n");
 518                return 1;
 519        }
 520
 521        if (!dhar_valid(pvt)) {
 522                edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this node %d\n",
 523                         pvt->mc_node_id);
 524                return 1;
 525        }
 526
 527        /* This node has Memory Hoisting */
 528
 529        /* +------------------+--------------------+--------------------+-----
 530         * | memory           | DRAM hole          | relocated          |
 531         * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
 532         * |                  |                    | DRAM hole          |
 533         * |                  |                    | [0x100000000,      |
 534         * |                  |                    |  (0x100000000+     |
 535         * |                  |                    |   (0xffffffff-x))] |
 536         * +------------------+--------------------+--------------------+-----
 537         *
 538         * Above is a diagram of physical memory showing the DRAM hole and the
 539         * relocated addresses from the DRAM hole.  As shown, the DRAM hole
 540         * starts at address x (the base address) and extends through address
 541         * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
 542         * addresses in the hole so that they start at 0x100000000.
 543         */
 544
 545        *hole_base = dhar_base(pvt);
 546        *hole_size = (1ULL << 32) - *hole_base;
 547
 548        *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
 549                                        : k8_dhar_offset(pvt);
 550
 551        edac_dbg(1, "  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
 552                 pvt->mc_node_id, (unsigned long)*hole_base,
 553                 (unsigned long)*hole_offset, (unsigned long)*hole_size);
 554
 555        return 0;
 556}
 557
 558#ifdef CONFIG_EDAC_DEBUG
 559#define EDAC_DCT_ATTR_SHOW(reg)                                         \
 560static ssize_t reg##_show(struct device *dev,                           \
 561                         struct device_attribute *mattr, char *data)    \
 562{                                                                       \
 563        struct mem_ctl_info *mci = to_mci(dev);                         \
 564        struct amd64_pvt *pvt = mci->pvt_info;                          \
 565                                                                        \
 566        return sprintf(data, "0x%016llx\n", (u64)pvt->reg);             \
 567}
 568
 569EDAC_DCT_ATTR_SHOW(dhar);
 570EDAC_DCT_ATTR_SHOW(dbam0);
 571EDAC_DCT_ATTR_SHOW(top_mem);
 572EDAC_DCT_ATTR_SHOW(top_mem2);
 573
 574static ssize_t hole_show(struct device *dev, struct device_attribute *mattr,
 575                         char *data)
 576{
 577        struct mem_ctl_info *mci = to_mci(dev);
 578
 579        u64 hole_base = 0;
 580        u64 hole_offset = 0;
 581        u64 hole_size = 0;
 582
 583        get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
 584
 585        return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
 586                                                 hole_size);
 587}
 588
 589/*
 590 * update NUM_DBG_ATTRS in case you add new members
 591 */
 592static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
 593static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
 594static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
 595static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
 596static DEVICE_ATTR(dram_hole, S_IRUGO, hole_show, NULL);
 597
 598static struct attribute *dbg_attrs[] = {
 599        &dev_attr_dhar.attr,
 600        &dev_attr_dbam.attr,
 601        &dev_attr_topmem.attr,
 602        &dev_attr_topmem2.attr,
 603        &dev_attr_dram_hole.attr,
 604        NULL
 605};
 606
 607static const struct attribute_group dbg_group = {
 608        .attrs = dbg_attrs,
 609};
 610
 611static ssize_t inject_section_show(struct device *dev,
 612                                   struct device_attribute *mattr, char *buf)
 613{
 614        struct mem_ctl_info *mci = to_mci(dev);
 615        struct amd64_pvt *pvt = mci->pvt_info;
 616        return sprintf(buf, "0x%x\n", pvt->injection.section);
 617}
 618
 619/*
 620 * store error injection section value which refers to one of 4 16-byte sections
 621 * within a 64-byte cacheline
 622 *
 623 * range: 0..3
 624 */
 625static ssize_t inject_section_store(struct device *dev,
 626                                    struct device_attribute *mattr,
 627                                    const char *data, size_t count)
 628{
 629        struct mem_ctl_info *mci = to_mci(dev);
 630        struct amd64_pvt *pvt = mci->pvt_info;
 631        unsigned long value;
 632        int ret;
 633
 634        ret = kstrtoul(data, 10, &value);
 635        if (ret < 0)
 636                return ret;
 637
 638        if (value > 3) {
 639                amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
 640                return -EINVAL;
 641        }
 642
 643        pvt->injection.section = (u32) value;
 644        return count;
 645}
 646
 647static ssize_t inject_word_show(struct device *dev,
 648                                struct device_attribute *mattr, char *buf)
 649{
 650        struct mem_ctl_info *mci = to_mci(dev);
 651        struct amd64_pvt *pvt = mci->pvt_info;
 652        return sprintf(buf, "0x%x\n", pvt->injection.word);
 653}
 654
 655/*
 656 * store error injection word value which refers to one of 9 16-bit word of the
 657 * 16-byte (128-bit + ECC bits) section
 658 *
 659 * range: 0..8
 660 */
 661static ssize_t inject_word_store(struct device *dev,
 662                                 struct device_attribute *mattr,
 663                                 const char *data, size_t count)
 664{
 665        struct mem_ctl_info *mci = to_mci(dev);
 666        struct amd64_pvt *pvt = mci->pvt_info;
 667        unsigned long value;
 668        int ret;
 669
 670        ret = kstrtoul(data, 10, &value);
 671        if (ret < 0)
 672                return ret;
 673
 674        if (value > 8) {
 675                amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
 676                return -EINVAL;
 677        }
 678
 679        pvt->injection.word = (u32) value;
 680        return count;
 681}
 682
 683static ssize_t inject_ecc_vector_show(struct device *dev,
 684                                      struct device_attribute *mattr,
 685                                      char *buf)
 686{
 687        struct mem_ctl_info *mci = to_mci(dev);
 688        struct amd64_pvt *pvt = mci->pvt_info;
 689        return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
 690}
 691
 692/*
 693 * store 16 bit error injection vector which enables injecting errors to the
 694 * corresponding bit within the error injection word above. When used during a
 695 * DRAM ECC read, it holds the contents of the of the DRAM ECC bits.
 696 */
 697static ssize_t inject_ecc_vector_store(struct device *dev,
 698                                       struct device_attribute *mattr,
 699                                       const char *data, size_t count)
 700{
 701        struct mem_ctl_info *mci = to_mci(dev);
 702        struct amd64_pvt *pvt = mci->pvt_info;
 703        unsigned long value;
 704        int ret;
 705
 706        ret = kstrtoul(data, 16, &value);
 707        if (ret < 0)
 708                return ret;
 709
 710        if (value & 0xFFFF0000) {
 711                amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
 712                return -EINVAL;
 713        }
 714
 715        pvt->injection.bit_map = (u32) value;
 716        return count;
 717}
 718
 719/*
 720 * Do a DRAM ECC read. Assemble staged values in the pvt area, format into
 721 * fields needed by the injection registers and read the NB Array Data Port.
 722 */
 723static ssize_t inject_read_store(struct device *dev,
 724                                 struct device_attribute *mattr,
 725                                 const char *data, size_t count)
 726{
 727        struct mem_ctl_info *mci = to_mci(dev);
 728        struct amd64_pvt *pvt = mci->pvt_info;
 729        unsigned long value;
 730        u32 section, word_bits;
 731        int ret;
 732
 733        ret = kstrtoul(data, 10, &value);
 734        if (ret < 0)
 735                return ret;
 736
 737        /* Form value to choose 16-byte section of cacheline */
 738        section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
 739
 740        amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
 741
 742        word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
 743
 744        /* Issue 'word' and 'bit' along with the READ request */
 745        amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
 746
 747        edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
 748
 749        return count;
 750}
 751
 752/*
 753 * Do a DRAM ECC write. Assemble staged values in the pvt area and format into
 754 * fields needed by the injection registers.
 755 */
 756static ssize_t inject_write_store(struct device *dev,
 757                                  struct device_attribute *mattr,
 758                                  const char *data, size_t count)
 759{
 760        struct mem_ctl_info *mci = to_mci(dev);
 761        struct amd64_pvt *pvt = mci->pvt_info;
 762        u32 section, word_bits, tmp;
 763        unsigned long value;
 764        int ret;
 765
 766        ret = kstrtoul(data, 10, &value);
 767        if (ret < 0)
 768                return ret;
 769
 770        /* Form value to choose 16-byte section of cacheline */
 771        section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
 772
 773        amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
 774
 775        word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
 776
 777        pr_notice_once("Don't forget to decrease MCE polling interval in\n"
 778                        "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
 779                        "so that you can get the error report faster.\n");
 780
 781        on_each_cpu(disable_caches, NULL, 1);
 782
 783        /* Issue 'word' and 'bit' along with the READ request */
 784        amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
 785
 786 retry:
 787        /* wait until injection happens */
 788        amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
 789        if (tmp & F10_NB_ARR_ECC_WR_REQ) {
 790                cpu_relax();
 791                goto retry;
 792        }
 793
 794        on_each_cpu(enable_caches, NULL, 1);
 795
 796        edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
 797
 798        return count;
 799}
 800
 801/*
 802 * update NUM_INJ_ATTRS in case you add new members
 803 */
 804
 805static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR,
 806                   inject_section_show, inject_section_store);
 807static DEVICE_ATTR(inject_word, S_IRUGO | S_IWUSR,
 808                   inject_word_show, inject_word_store);
 809static DEVICE_ATTR(inject_ecc_vector, S_IRUGO | S_IWUSR,
 810                   inject_ecc_vector_show, inject_ecc_vector_store);
 811static DEVICE_ATTR(inject_write, S_IWUSR,
 812                   NULL, inject_write_store);
 813static DEVICE_ATTR(inject_read,  S_IWUSR,
 814                   NULL, inject_read_store);
 815
 816static struct attribute *inj_attrs[] = {
 817        &dev_attr_inject_section.attr,
 818        &dev_attr_inject_word.attr,
 819        &dev_attr_inject_ecc_vector.attr,
 820        &dev_attr_inject_write.attr,
 821        &dev_attr_inject_read.attr,
 822        NULL
 823};
 824
 825static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
 826{
 827        struct device *dev = kobj_to_dev(kobj);
 828        struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
 829        struct amd64_pvt *pvt = mci->pvt_info;
 830
 831        /* Families which have that injection hw */
 832        if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
 833                return attr->mode;
 834
 835        return 0;
 836}
 837
 838static const struct attribute_group inj_group = {
 839        .attrs = inj_attrs,
 840        .is_visible = inj_is_visible,
 841};
 842#endif /* CONFIG_EDAC_DEBUG */
 843
 844/*
 845 * Return the DramAddr that the SysAddr given by @sys_addr maps to.  It is
 846 * assumed that sys_addr maps to the node given by mci.
 847 *
 848 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
 849 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
 850 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
 851 * then it is also involved in translating a SysAddr to a DramAddr. Sections
 852 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
 853 * These parts of the documentation are unclear. I interpret them as follows:
 854 *
 855 * When node n receives a SysAddr, it processes the SysAddr as follows:
 856 *
 857 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
 858 *    Limit registers for node n. If the SysAddr is not within the range
 859 *    specified by the base and limit values, then node n ignores the Sysaddr
 860 *    (since it does not map to node n). Otherwise continue to step 2 below.
 861 *
 862 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
 863 *    disabled so skip to step 3 below. Otherwise see if the SysAddr is within
 864 *    the range of relocated addresses (starting at 0x100000000) from the DRAM
 865 *    hole. If not, skip to step 3 below. Else get the value of the
 866 *    DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
 867 *    offset defined by this value from the SysAddr.
 868 *
 869 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
 870 *    Base register for node n. To obtain the DramAddr, subtract the base
 871 *    address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
 872 */
 873static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
 874{
 875        struct amd64_pvt *pvt = mci->pvt_info;
 876        u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
 877        int ret;
 878
 879        dram_base = get_dram_base(pvt, pvt->mc_node_id);
 880
 881        ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
 882        if (!ret) {
 883                if ((sys_addr >= (1ULL << 32)) &&
 884                    (sys_addr < ((1ULL << 32) + hole_size))) {
 885                        /* use DHAR to translate SysAddr to DramAddr */
 886                        dram_addr = sys_addr - hole_offset;
 887
 888                        edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
 889                                 (unsigned long)sys_addr,
 890                                 (unsigned long)dram_addr);
 891
 892                        return dram_addr;
 893                }
 894        }
 895
 896        /*
 897         * Translate the SysAddr to a DramAddr as shown near the start of
 898         * section 3.4.4 (p. 70).  Although sys_addr is a 64-bit value, the k8
 899         * only deals with 40-bit values.  Therefore we discard bits 63-40 of
 900         * sys_addr below.  If bit 39 of sys_addr is 1 then the bits we
 901         * discard are all 1s.  Otherwise the bits we discard are all 0s.  See
 902         * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
 903         * Programmer's Manual Volume 1 Application Programming.
 904         */
 905        dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
 906
 907        edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
 908                 (unsigned long)sys_addr, (unsigned long)dram_addr);
 909        return dram_addr;
 910}
 911
 912/*
 913 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
 914 * (section 3.4.4.1).  Return the number of bits from a SysAddr that are used
 915 * for node interleaving.
 916 */
 917static int num_node_interleave_bits(unsigned intlv_en)
 918{
 919        static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
 920        int n;
 921
 922        BUG_ON(intlv_en > 7);
 923        n = intlv_shift_table[intlv_en];
 924        return n;
 925}
 926
 927/* Translate the DramAddr given by @dram_addr to an InputAddr. */
 928static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
 929{
 930        struct amd64_pvt *pvt;
 931        int intlv_shift;
 932        u64 input_addr;
 933
 934        pvt = mci->pvt_info;
 935
 936        /*
 937         * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
 938         * concerning translating a DramAddr to an InputAddr.
 939         */
 940        intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
 941        input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
 942                      (dram_addr & 0xfff);
 943
 944        edac_dbg(2, "  Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
 945                 intlv_shift, (unsigned long)dram_addr,
 946                 (unsigned long)input_addr);
 947
 948        return input_addr;
 949}
 950
 951/*
 952 * Translate the SysAddr represented by @sys_addr to an InputAddr.  It is
 953 * assumed that @sys_addr maps to the node given by mci.
 954 */
 955static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
 956{
 957        u64 input_addr;
 958
 959        input_addr =
 960            dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
 961
 962        edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
 963                 (unsigned long)sys_addr, (unsigned long)input_addr);
 964
 965        return input_addr;
 966}
 967
 968/* Map the Error address to a PAGE and PAGE OFFSET. */
 969static inline void error_address_to_page_and_offset(u64 error_address,
 970                                                    struct err_info *err)
 971{
 972        err->page = (u32) (error_address >> PAGE_SHIFT);
 973        err->offset = ((u32) error_address) & ~PAGE_MASK;
 974}
 975
 976/*
 977 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
 978 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
 979 * of a node that detected an ECC memory error.  mci represents the node that
 980 * the error address maps to (possibly different from the node that detected
 981 * the error).  Return the number of the csrow that sys_addr maps to, or -1 on
 982 * error.
 983 */
 984static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
 985{
 986        int csrow;
 987
 988        csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
 989
 990        if (csrow == -1)
 991                amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
 992                                  "address 0x%lx\n", (unsigned long)sys_addr);
 993        return csrow;
 994}
 995
 996static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
 997
 998/*
 999 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
1000 * are ECC capable.
1001 */
1002static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
1003{
1004        unsigned long edac_cap = EDAC_FLAG_NONE;
1005        u8 bit;
1006
1007        if (pvt->umc) {
1008                u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
1009
1010                for_each_umc(i) {
1011                        if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1012                                continue;
1013
1014                        umc_en_mask |= BIT(i);
1015
1016                        /* UMC Configuration bit 12 (DimmEccEn) */
1017                        if (pvt->umc[i].umc_cfg & BIT(12))
1018                                dimm_ecc_en_mask |= BIT(i);
1019                }
1020
1021                if (umc_en_mask == dimm_ecc_en_mask)
1022                        edac_cap = EDAC_FLAG_SECDED;
1023        } else {
1024                bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1025                        ? 19
1026                        : 17;
1027
1028                if (pvt->dclr0 & BIT(bit))
1029                        edac_cap = EDAC_FLAG_SECDED;
1030        }
1031
1032        return edac_cap;
1033}
1034
1035static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
1036
1037static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
1038{
1039        edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
1040
1041        if (pvt->dram_type == MEM_LRDDR3) {
1042                u32 dcsm = pvt->csels[chan].csmasks[0];
1043                /*
1044                 * It's assumed all LRDIMMs in a DCT are going to be of
1045                 * same 'type' until proven otherwise. So, use a cs
1046                 * value of '0' here to get dcsm value.
1047                 */
1048                edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1049        }
1050
1051        edac_dbg(1, "All DIMMs support ECC:%s\n",
1052                    (dclr & BIT(19)) ? "yes" : "no");
1053
1054
1055        edac_dbg(1, "  PAR/ERR parity: %s\n",
1056                 (dclr & BIT(8)) ?  "enabled" : "disabled");
1057
1058        if (pvt->fam == 0x10)
1059                edac_dbg(1, "  DCT 128bit mode width: %s\n",
1060                         (dclr & BIT(11)) ?  "128b" : "64b");
1061
1062        edac_dbg(1, "  x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1063                 (dclr & BIT(12)) ?  "yes" : "no",
1064                 (dclr & BIT(13)) ?  "yes" : "no",
1065                 (dclr & BIT(14)) ?  "yes" : "no",
1066                 (dclr & BIT(15)) ?  "yes" : "no");
1067}
1068
1069#define CS_EVEN_PRIMARY         BIT(0)
1070#define CS_ODD_PRIMARY          BIT(1)
1071#define CS_EVEN_SECONDARY       BIT(2)
1072#define CS_ODD_SECONDARY        BIT(3)
1073
1074#define CS_EVEN                 (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1075#define CS_ODD                  (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1076
1077static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
1078{
1079        int cs_mode = 0;
1080
1081        if (csrow_enabled(2 * dimm, ctrl, pvt))
1082                cs_mode |= CS_EVEN_PRIMARY;
1083
1084        if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1085                cs_mode |= CS_ODD_PRIMARY;
1086
1087        /* Asymmetric dual-rank DIMM support. */
1088        if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
1089                cs_mode |= CS_ODD_SECONDARY;
1090
1091        return cs_mode;
1092}
1093
1094static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
1095{
1096        int dimm, size0, size1, cs0, cs1, cs_mode;
1097
1098        edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
1099
1100        for (dimm = 0; dimm < 2; dimm++) {
1101                cs0 = dimm * 2;
1102                cs1 = dimm * 2 + 1;
1103
1104                cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
1105
1106                size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
1107                size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
1108
1109                amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1110                                cs0,    size0,
1111                                cs1,    size1);
1112        }
1113}
1114
1115static void __dump_misc_regs_df(struct amd64_pvt *pvt)
1116{
1117        struct amd64_umc *umc;
1118        u32 i, tmp, umc_base;
1119
1120        for_each_umc(i) {
1121                umc_base = get_umc_base(i);
1122                umc = &pvt->umc[i];
1123
1124                edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
1125                edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
1126                edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
1127                edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
1128
1129                amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
1130                edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
1131
1132                amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
1133                edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
1134                edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
1135
1136                edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1137                                i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
1138                                    (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
1139                edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1140                                i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
1141                edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1142                                i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
1143                edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1144                                i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
1145
1146                if (pvt->dram_type == MEM_LRDDR4) {
1147                        amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
1148                        edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1149                                        i, 1 << ((tmp >> 4) & 0x3));
1150                }
1151
1152                debug_display_dimm_sizes_df(pvt, i);
1153        }
1154
1155        edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
1156                 pvt->dhar, dhar_base(pvt));
1157}
1158
1159/* Display and decode various NB registers for debug purposes. */
1160static void __dump_misc_regs(struct amd64_pvt *pvt)
1161{
1162        edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
1163
1164        edac_dbg(1, "  NB two channel DRAM capable: %s\n",
1165                 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
1166
1167        edac_dbg(1, "  ECC capable: %s, ChipKill ECC capable: %s\n",
1168                 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
1169                 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
1170
1171        debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
1172
1173        edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
1174
1175        edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1176                 pvt->dhar, dhar_base(pvt),
1177                 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1178                                   : f10_dhar_offset(pvt));
1179
1180        debug_display_dimm_sizes(pvt, 0);
1181
1182        /* everything below this point is Fam10h and above */
1183        if (pvt->fam == 0xf)
1184                return;
1185
1186        debug_display_dimm_sizes(pvt, 1);
1187
1188        /* Only if NOT ganged does dclr1 have valid info */
1189        if (!dct_ganging_enabled(pvt))
1190                debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
1191}
1192
1193/* Display and decode various NB registers for debug purposes. */
1194static void dump_misc_regs(struct amd64_pvt *pvt)
1195{
1196        if (pvt->umc)
1197                __dump_misc_regs_df(pvt);
1198        else
1199                __dump_misc_regs(pvt);
1200
1201        edac_dbg(1, "  DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
1202
1203        amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
1204}
1205
1206/*
1207 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
1208 */
1209static void prep_chip_selects(struct amd64_pvt *pvt)
1210{
1211        if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
1212                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1213                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
1214        } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
1215                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
1216                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
1217        } else if (pvt->fam >= 0x17) {
1218                int umc;
1219
1220                for_each_umc(umc) {
1221                        pvt->csels[umc].b_cnt = 4;
1222                        pvt->csels[umc].m_cnt = 2;
1223                }
1224
1225        } else {
1226                pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1227                pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
1228        }
1229}
1230
1231static void read_umc_base_mask(struct amd64_pvt *pvt)
1232{
1233        u32 umc_base_reg, umc_base_reg_sec;
1234        u32 umc_mask_reg, umc_mask_reg_sec;
1235        u32 base_reg, base_reg_sec;
1236        u32 mask_reg, mask_reg_sec;
1237        u32 *base, *base_sec;
1238        u32 *mask, *mask_sec;
1239        int cs, umc;
1240
1241        for_each_umc(umc) {
1242                umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
1243                umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
1244
1245                for_each_chip_select(cs, umc, pvt) {
1246                        base = &pvt->csels[umc].csbases[cs];
1247                        base_sec = &pvt->csels[umc].csbases_sec[cs];
1248
1249                        base_reg = umc_base_reg + (cs * 4);
1250                        base_reg_sec = umc_base_reg_sec + (cs * 4);
1251
1252                        if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1253                                edac_dbg(0, "  DCSB%d[%d]=0x%08x reg: 0x%x\n",
1254                                         umc, cs, *base, base_reg);
1255
1256                        if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
1257                                edac_dbg(0, "    DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1258                                         umc, cs, *base_sec, base_reg_sec);
1259                }
1260
1261                umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
1262                umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;
1263
1264                for_each_chip_select_mask(cs, umc, pvt) {
1265                        mask = &pvt->csels[umc].csmasks[cs];
1266                        mask_sec = &pvt->csels[umc].csmasks_sec[cs];
1267
1268                        mask_reg = umc_mask_reg + (cs * 4);
1269                        mask_reg_sec = umc_mask_reg_sec + (cs * 4);
1270
1271                        if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1272                                edac_dbg(0, "  DCSM%d[%d]=0x%08x reg: 0x%x\n",
1273                                         umc, cs, *mask, mask_reg);
1274
1275                        if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
1276                                edac_dbg(0, "    DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1277                                         umc, cs, *mask_sec, mask_reg_sec);
1278                }
1279        }
1280}
1281
1282/*
1283 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
1284 */
1285static void read_dct_base_mask(struct amd64_pvt *pvt)
1286{
1287        int cs;
1288
1289        prep_chip_selects(pvt);
1290
1291        if (pvt->umc)
1292                return read_umc_base_mask(pvt);
1293
1294        for_each_chip_select(cs, 0, pvt) {
1295                int reg0   = DCSB0 + (cs * 4);
1296                int reg1   = DCSB1 + (cs * 4);
1297                u32 *base0 = &pvt->csels[0].csbases[cs];
1298                u32 *base1 = &pvt->csels[1].csbases[cs];
1299
1300                if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1301                        edac_dbg(0, "  DCSB0[%d]=0x%08x reg: F2x%x\n",
1302                                 cs, *base0, reg0);
1303
1304                if (pvt->fam == 0xf)
1305                        continue;
1306
1307                if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1308                        edac_dbg(0, "  DCSB1[%d]=0x%08x reg: F2x%x\n",
1309                                 cs, *base1, (pvt->fam == 0x10) ? reg1
1310                                                        : reg0);
1311        }
1312
1313        for_each_chip_select_mask(cs, 0, pvt) {
1314                int reg0   = DCSM0 + (cs * 4);
1315                int reg1   = DCSM1 + (cs * 4);
1316                u32 *mask0 = &pvt->csels[0].csmasks[cs];
1317                u32 *mask1 = &pvt->csels[1].csmasks[cs];
1318
1319                if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1320                        edac_dbg(0, "    DCSM0[%d]=0x%08x reg: F2x%x\n",
1321                                 cs, *mask0, reg0);
1322
1323                if (pvt->fam == 0xf)
1324                        continue;
1325
1326                if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1327                        edac_dbg(0, "    DCSM1[%d]=0x%08x reg: F2x%x\n",
1328                                 cs, *mask1, (pvt->fam == 0x10) ? reg1
1329                                                        : reg0);
1330        }
1331}
1332
1333static void determine_memory_type(struct amd64_pvt *pvt)
1334{
1335        u32 dram_ctrl, dcsm;
1336
1337        if (pvt->umc) {
1338                if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1339                        pvt->dram_type = MEM_LRDDR4;
1340                else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1341                        pvt->dram_type = MEM_RDDR4;
1342                else
1343                        pvt->dram_type = MEM_DDR4;
1344                return;
1345        }
1346
1347        switch (pvt->fam) {
1348        case 0xf:
1349                if (pvt->ext_model >= K8_REV_F)
1350                        goto ddr3;
1351
1352                pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1353                return;
1354
1355        case 0x10:
1356                if (pvt->dchr0 & DDR3_MODE)
1357                        goto ddr3;
1358
1359                pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1360                return;
1361
1362        case 0x15:
1363                if (pvt->model < 0x60)
1364                        goto ddr3;
1365
1366                /*
1367                 * Model 0x60h needs special handling:
1368                 *
1369                 * We use a Chip Select value of '0' to obtain dcsm.
1370                 * Theoretically, it is possible to populate LRDIMMs of different
1371                 * 'Rank' value on a DCT. But this is not the common case. So,
1372                 * it's reasonable to assume all DIMMs are going to be of same
1373                 * 'type' until proven otherwise.
1374                 */
1375                amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1376                dcsm = pvt->csels[0].csmasks[0];
1377
1378                if (((dram_ctrl >> 8) & 0x7) == 0x2)
1379                        pvt->dram_type = MEM_DDR4;
1380                else if (pvt->dclr0 & BIT(16))
1381                        pvt->dram_type = MEM_DDR3;
1382                else if (dcsm & 0x3)
1383                        pvt->dram_type = MEM_LRDDR3;
1384                else
1385                        pvt->dram_type = MEM_RDDR3;
1386
1387                return;
1388
1389        case 0x16:
1390                goto ddr3;
1391
1392        default:
1393                WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1394                pvt->dram_type = MEM_EMPTY;
1395        }
1396        return;
1397
1398ddr3:
1399        pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1400}
1401
1402/* Get the number of DCT channels the memory controller is using. */
1403static int k8_early_channel_count(struct amd64_pvt *pvt)
1404{
1405        int flag;
1406
1407        if (pvt->ext_model >= K8_REV_F)
1408                /* RevF (NPT) and later */
1409                flag = pvt->dclr0 & WIDTH_128;
1410        else
1411                /* RevE and earlier */
1412                flag = pvt->dclr0 & REVE_WIDTH_128;
1413
1414        /* not used */
1415        pvt->dclr1 = 0;
1416
1417        return (flag) ? 2 : 1;
1418}
1419
1420/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
1421static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1422{
1423        u16 mce_nid = topology_die_id(m->extcpu);
1424        struct mem_ctl_info *mci;
1425        u8 start_bit = 1;
1426        u8 end_bit   = 47;
1427        u64 addr;
1428
1429        mci = edac_mc_find(mce_nid);
1430        if (!mci)
1431                return 0;
1432
1433        pvt = mci->pvt_info;
1434
1435        if (pvt->fam == 0xf) {
1436                start_bit = 3;
1437                end_bit   = 39;
1438        }
1439
1440        addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1441
1442        /*
1443         * Erratum 637 workaround
1444         */
1445        if (pvt->fam == 0x15) {
1446                u64 cc6_base, tmp_addr;
1447                u32 tmp;
1448                u8 intlv_en;
1449
1450                if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1451                        return addr;
1452
1453
1454                amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1455                intlv_en = tmp >> 21 & 0x7;
1456
1457                /* add [47:27] + 3 trailing bits */
1458                cc6_base  = (tmp & GENMASK_ULL(20, 0)) << 3;
1459
1460                /* reverse and add DramIntlvEn */
1461                cc6_base |= intlv_en ^ 0x7;
1462
1463                /* pin at [47:24] */
1464                cc6_base <<= 24;
1465
1466                if (!intlv_en)
1467                        return cc6_base | (addr & GENMASK_ULL(23, 0));
1468
1469                amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1470
1471                                                        /* faster log2 */
1472                tmp_addr  = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1473
1474                /* OR DramIntlvSel into bits [14:12] */
1475                tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1476
1477                /* add remaining [11:0] bits from original MC4_ADDR */
1478                tmp_addr |= addr & GENMASK_ULL(11, 0);
1479
1480                return cc6_base | tmp_addr;
1481        }
1482
1483        return addr;
1484}
1485
1486static struct pci_dev *pci_get_related_function(unsigned int vendor,
1487                                                unsigned int device,
1488                                                struct pci_dev *related)
1489{
1490        struct pci_dev *dev = NULL;
1491
1492        while ((dev = pci_get_device(vendor, device, dev))) {
1493                if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1494                    (dev->bus->number == related->bus->number) &&
1495                    (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1496                        break;
1497        }
1498
1499        return dev;
1500}
1501
1502static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1503{
1504        struct amd_northbridge *nb;
1505        struct pci_dev *f1 = NULL;
1506        unsigned int pci_func;
1507        int off = range << 3;
1508        u32 llim;
1509
1510        amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off,  &pvt->ranges[range].base.lo);
1511        amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1512
1513        if (pvt->fam == 0xf)
1514                return;
1515
1516        if (!dram_rw(pvt, range))
1517                return;
1518
1519        amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off,  &pvt->ranges[range].base.hi);
1520        amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1521
1522        /* F15h: factor in CC6 save area by reading dst node's limit reg */
1523        if (pvt->fam != 0x15)
1524                return;
1525
1526        nb = node_to_amd_nb(dram_dst_node(pvt, range));
1527        if (WARN_ON(!nb))
1528                return;
1529
1530        if (pvt->model == 0x60)
1531                pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1532        else if (pvt->model == 0x30)
1533                pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1534        else
1535                pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1536
1537        f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1538        if (WARN_ON(!f1))
1539                return;
1540
1541        amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1542
1543        pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1544
1545                                    /* {[39:27],111b} */
1546        pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1547
1548        pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1549
1550                                    /* [47:40] */
1551        pvt->ranges[range].lim.hi |= llim >> 13;
1552
1553        pci_dev_put(f1);
1554}
1555
1556static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1557                                    struct err_info *err)
1558{
1559        struct amd64_pvt *pvt = mci->pvt_info;
1560
1561        error_address_to_page_and_offset(sys_addr, err);
1562
1563        /*
1564         * Find out which node the error address belongs to. This may be
1565         * different from the node that detected the error.
1566         */
1567        err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1568        if (!err->src_mci) {
1569                amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1570                             (unsigned long)sys_addr);
1571                err->err_code = ERR_NODE;
1572                return;
1573        }
1574
1575        /* Now map the sys_addr to a CSROW */
1576        err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1577        if (err->csrow < 0) {
1578                err->err_code = ERR_CSROW;
1579                return;
1580        }
1581
1582        /* CHIPKILL enabled */
1583        if (pvt->nbcfg & NBCFG_CHIPKILL) {
1584                err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1585                if (err->channel < 0) {
1586                        /*
1587                         * Syndrome didn't map, so we don't know which of the
1588                         * 2 DIMMs is in error. So we need to ID 'both' of them
1589                         * as suspect.
1590                         */
1591                        amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1592                                      "possible error reporting race\n",
1593                                      err->syndrome);
1594                        err->err_code = ERR_CHANNEL;
1595                        return;
1596                }
1597        } else {
1598                /*
1599                 * non-chipkill ecc mode
1600                 *
1601                 * The k8 documentation is unclear about how to determine the
1602                 * channel number when using non-chipkill memory.  This method
1603                 * was obtained from email communication with someone at AMD.
1604                 * (Wish the email was placed in this comment - norsk)
1605                 */
1606                err->channel = ((sys_addr & BIT(3)) != 0);
1607        }
1608}
1609
1610static int ddr2_cs_size(unsigned i, bool dct_width)
1611{
1612        unsigned shift = 0;
1613
1614        if (i <= 2)
1615                shift = i;
1616        else if (!(i & 0x1))
1617                shift = i >> 1;
1618        else
1619                shift = (i + 1) >> 1;
1620
1621        return 128 << (shift + !!dct_width);
1622}
1623
1624static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1625                                  unsigned cs_mode, int cs_mask_nr)
1626{
1627        u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1628
1629        if (pvt->ext_model >= K8_REV_F) {
1630                WARN_ON(cs_mode > 11);
1631                return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1632        }
1633        else if (pvt->ext_model >= K8_REV_D) {
1634                unsigned diff;
1635                WARN_ON(cs_mode > 10);
1636
1637                /*
1638                 * the below calculation, besides trying to win an obfuscated C
1639                 * contest, maps cs_mode values to DIMM chip select sizes. The
1640                 * mappings are:
1641                 *
1642                 * cs_mode      CS size (mb)
1643                 * =======      ============
1644                 * 0            32
1645                 * 1            64
1646                 * 2            128
1647                 * 3            128
1648                 * 4            256
1649                 * 5            512
1650                 * 6            256
1651                 * 7            512
1652                 * 8            1024
1653                 * 9            1024
1654                 * 10           2048
1655                 *
1656                 * Basically, it calculates a value with which to shift the
1657                 * smallest CS size of 32MB.
1658                 *
1659                 * ddr[23]_cs_size have a similar purpose.
1660                 */
1661                diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1662
1663                return 32 << (cs_mode - diff);
1664        }
1665        else {
1666                WARN_ON(cs_mode > 6);
1667                return 32 << cs_mode;
1668        }
1669}
1670
1671/*
1672 * Get the number of DCT channels in use.
1673 *
1674 * Return:
1675 *      number of Memory Channels in operation
1676 * Pass back:
1677 *      contents of the DCL0_LOW register
1678 */
1679static int f1x_early_channel_count(struct amd64_pvt *pvt)
1680{
1681        int i, j, channels = 0;
1682
1683        /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1684        if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1685                return 2;
1686
1687        /*
1688         * Need to check if in unganged mode: In such, there are 2 channels,
1689         * but they are not in 128 bit mode and thus the above 'dclr0' status
1690         * bit will be OFF.
1691         *
1692         * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1693         * their CSEnable bit on. If so, then SINGLE DIMM case.
1694         */
1695        edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1696
1697        /*
1698         * Check DRAM Bank Address Mapping values for each DIMM to see if there
1699         * is more than just one DIMM present in unganged mode. Need to check
1700         * both controllers since DIMMs can be placed in either one.
1701         */
1702        for (i = 0; i < 2; i++) {
1703                u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1704
1705                for (j = 0; j < 4; j++) {
1706                        if (DBAM_DIMM(j, dbam) > 0) {
1707                                channels++;
1708                                break;
1709                        }
1710                }
1711        }
1712
1713        if (channels > 2)
1714                channels = 2;
1715
1716        amd64_info("MCT channel count: %d\n", channels);
1717
1718        return channels;
1719}
1720
1721static int f17_early_channel_count(struct amd64_pvt *pvt)
1722{
1723        int i, channels = 0;
1724
1725        /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
1726        for_each_umc(i)
1727                channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1728
1729        amd64_info("MCT channel count: %d\n", channels);
1730
1731        return channels;
1732}
1733
1734static int ddr3_cs_size(unsigned i, bool dct_width)
1735{
1736        unsigned shift = 0;
1737        int cs_size = 0;
1738
1739        if (i == 0 || i == 3 || i == 4)
1740                cs_size = -1;
1741        else if (i <= 2)
1742                shift = i;
1743        else if (i == 12)
1744                shift = 7;
1745        else if (!(i & 0x1))
1746                shift = i >> 1;
1747        else
1748                shift = (i + 1) >> 1;
1749
1750        if (cs_size != -1)
1751                cs_size = (128 * (1 << !!dct_width)) << shift;
1752
1753        return cs_size;
1754}
1755
1756static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1757{
1758        unsigned shift = 0;
1759        int cs_size = 0;
1760
1761        if (i < 4 || i == 6)
1762                cs_size = -1;
1763        else if (i == 12)
1764                shift = 7;
1765        else if (!(i & 0x1))
1766                shift = i >> 1;
1767        else
1768                shift = (i + 1) >> 1;
1769
1770        if (cs_size != -1)
1771                cs_size = rank_multiply * (128 << shift);
1772
1773        return cs_size;
1774}
1775
1776static int ddr4_cs_size(unsigned i)
1777{
1778        int cs_size = 0;
1779
1780        if (i == 0)
1781                cs_size = -1;
1782        else if (i == 1)
1783                cs_size = 1024;
1784        else
1785                /* Min cs_size = 1G */
1786                cs_size = 1024 * (1 << (i >> 1));
1787
1788        return cs_size;
1789}
1790
1791static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1792                                   unsigned cs_mode, int cs_mask_nr)
1793{
1794        u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1795
1796        WARN_ON(cs_mode > 11);
1797
1798        if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1799                return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1800        else
1801                return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1802}
1803
1804/*
1805 * F15h supports only 64bit DCT interfaces
1806 */
1807static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1808                                   unsigned cs_mode, int cs_mask_nr)
1809{
1810        WARN_ON(cs_mode > 12);
1811
1812        return ddr3_cs_size(cs_mode, false);
1813}
1814
1815/* F15h M60h supports DDR4 mapping as well.. */
1816static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1817                                        unsigned cs_mode, int cs_mask_nr)
1818{
1819        int cs_size;
1820        u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1821
1822        WARN_ON(cs_mode > 12);
1823
1824        if (pvt->dram_type == MEM_DDR4) {
1825                if (cs_mode > 9)
1826                        return -1;
1827
1828                cs_size = ddr4_cs_size(cs_mode);
1829        } else if (pvt->dram_type == MEM_LRDDR3) {
1830                unsigned rank_multiply = dcsm & 0xf;
1831
1832                if (rank_multiply == 3)
1833                        rank_multiply = 4;
1834                cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1835        } else {
1836                /* Minimum cs size is 512mb for F15hM60h*/
1837                if (cs_mode == 0x1)
1838                        return -1;
1839
1840                cs_size = ddr3_cs_size(cs_mode, false);
1841        }
1842
1843        return cs_size;
1844}
1845
1846/*
1847 * F16h and F15h model 30h have only limited cs_modes.
1848 */
1849static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1850                                unsigned cs_mode, int cs_mask_nr)
1851{
1852        WARN_ON(cs_mode > 12);
1853
1854        if (cs_mode == 6 || cs_mode == 8 ||
1855            cs_mode == 9 || cs_mode == 12)
1856                return -1;
1857        else
1858                return ddr3_cs_size(cs_mode, false);
1859}
1860
1861static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1862                                    unsigned int cs_mode, int csrow_nr)
1863{
1864        u32 addr_mask_orig, addr_mask_deinterleaved;
1865        u32 msb, weight, num_zero_bits;
1866        int dimm, size = 0;
1867
1868        /* No Chip Selects are enabled. */
1869        if (!cs_mode)
1870                return size;
1871
1872        /* Requested size of an even CS but none are enabled. */
1873        if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
1874                return size;
1875
1876        /* Requested size of an odd CS but none are enabled. */
1877        if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
1878                return size;
1879
1880        /*
1881         * There is one mask per DIMM, and two Chip Selects per DIMM.
1882         *      CS0 and CS1 -> DIMM0
1883         *      CS2 and CS3 -> DIMM1
1884         */
1885        dimm = csrow_nr >> 1;
1886
1887        /* Asymmetric dual-rank DIMM support. */
1888        if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
1889                addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
1890        else
1891                addr_mask_orig = pvt->csels[umc].csmasks[dimm];
1892
1893        /*
1894         * The number of zero bits in the mask is equal to the number of bits
1895         * in a full mask minus the number of bits in the current mask.
1896         *
1897         * The MSB is the number of bits in the full mask because BIT[0] is
1898         * always 0.
1899         */
1900        msb = fls(addr_mask_orig) - 1;
1901        weight = hweight_long(addr_mask_orig);
1902        num_zero_bits = msb - weight;
1903
1904        /* Take the number of zero bits off from the top of the mask. */
1905        addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
1906
1907        edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
1908        edac_dbg(1, "  Original AddrMask: 0x%x\n", addr_mask_orig);
1909        edac_dbg(1, "  Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
1910
1911        /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1912        size = (addr_mask_deinterleaved >> 2) + 1;
1913
1914        /* Return size in MBs. */
1915        return size >> 10;
1916}
1917
1918static void read_dram_ctl_register(struct amd64_pvt *pvt)
1919{
1920
1921        if (pvt->fam == 0xf)
1922                return;
1923
1924        if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1925                edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1926                         pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1927
1928                edac_dbg(0, "  DCTs operate in %s mode\n",
1929                         (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1930
1931                if (!dct_ganging_enabled(pvt))
1932                        edac_dbg(0, "  Address range split per DCT: %s\n",
1933                                 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1934
1935                edac_dbg(0, "  data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1936                         (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1937                         (dct_memory_cleared(pvt) ? "yes" : "no"));
1938
1939                edac_dbg(0, "  channel interleave: %s, "
1940                         "interleave bits selector: 0x%x\n",
1941                         (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1942                         dct_sel_interleave_addr(pvt));
1943        }
1944
1945        amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1946}
1947
1948/*
1949 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1950 * 2.10.12 Memory Interleaving Modes).
1951 */
1952static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1953                                     u8 intlv_en, int num_dcts_intlv,
1954                                     u32 dct_sel)
1955{
1956        u8 channel = 0;
1957        u8 select;
1958
1959        if (!(intlv_en))
1960                return (u8)(dct_sel);
1961
1962        if (num_dcts_intlv == 2) {
1963                select = (sys_addr >> 8) & 0x3;
1964                channel = select ? 0x3 : 0;
1965        } else if (num_dcts_intlv == 4) {
1966                u8 intlv_addr = dct_sel_interleave_addr(pvt);
1967                switch (intlv_addr) {
1968                case 0x4:
1969                        channel = (sys_addr >> 8) & 0x3;
1970                        break;
1971                case 0x5:
1972                        channel = (sys_addr >> 9) & 0x3;
1973                        break;
1974                }
1975        }
1976        return channel;
1977}
1978
1979/*
1980 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1981 * Interleaving Modes.
1982 */
1983static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1984                                bool hi_range_sel, u8 intlv_en)
1985{
1986        u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1987
1988        if (dct_ganging_enabled(pvt))
1989                return 0;
1990
1991        if (hi_range_sel)
1992                return dct_sel_high;
1993
1994        /*
1995         * see F2x110[DctSelIntLvAddr] - channel interleave mode
1996         */
1997        if (dct_interleave_enabled(pvt)) {
1998                u8 intlv_addr = dct_sel_interleave_addr(pvt);
1999
2000                /* return DCT select function: 0=DCT0, 1=DCT1 */
2001                if (!intlv_addr)
2002                        return sys_addr >> 6 & 1;
2003
2004                if (intlv_addr & 0x2) {
2005                        u8 shift = intlv_addr & 0x1 ? 9 : 6;
2006                        u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
2007
2008                        return ((sys_addr >> shift) & 1) ^ temp;
2009                }
2010
2011                if (intlv_addr & 0x4) {
2012                        u8 shift = intlv_addr & 0x1 ? 9 : 8;
2013
2014                        return (sys_addr >> shift) & 1;
2015                }
2016
2017                return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2018        }
2019
2020        if (dct_high_range_enabled(pvt))
2021                return ~dct_sel_high & 1;
2022
2023        return 0;
2024}
2025
2026/* Convert the sys_addr to the normalized DCT address */
2027static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
2028                                 u64 sys_addr, bool hi_rng,
2029                                 u32 dct_sel_base_addr)
2030{
2031        u64 chan_off;
2032        u64 dram_base           = get_dram_base(pvt, range);
2033        u64 hole_off            = f10_dhar_offset(pvt);
2034        u64 dct_sel_base_off    = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
2035
2036        if (hi_rng) {
2037                /*
2038                 * if
2039                 * base address of high range is below 4Gb
2040                 * (bits [47:27] at [31:11])
2041                 * DRAM address space on this DCT is hoisted above 4Gb  &&
2042                 * sys_addr > 4Gb
2043                 *
2044                 *      remove hole offset from sys_addr
2045                 * else
2046                 *      remove high range offset from sys_addr
2047                 */
2048                if ((!(dct_sel_base_addr >> 16) ||
2049                     dct_sel_base_addr < dhar_base(pvt)) &&
2050                    dhar_valid(pvt) &&
2051                    (sys_addr >= BIT_64(32)))
2052                        chan_off = hole_off;
2053                else
2054                        chan_off = dct_sel_base_off;
2055        } else {
2056                /*
2057                 * if
2058                 * we have a valid hole         &&
2059                 * sys_addr > 4Gb
2060                 *
2061                 *      remove hole
2062                 * else
2063                 *      remove dram base to normalize to DCT address
2064                 */
2065                if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
2066                        chan_off = hole_off;
2067                else
2068                        chan_off = dram_base;
2069        }
2070
2071        return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
2072}
2073
2074/*
2075 * checks if the csrow passed in is marked as SPARED, if so returns the new
2076 * spare row
2077 */
2078static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
2079{
2080        int tmp_cs;
2081
2082        if (online_spare_swap_done(pvt, dct) &&
2083            csrow == online_spare_bad_dramcs(pvt, dct)) {
2084
2085                for_each_chip_select(tmp_cs, dct, pvt) {
2086                        if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2087                                csrow = tmp_cs;
2088                                break;
2089                        }
2090                }
2091        }
2092        return csrow;
2093}
2094
2095/*
2096 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
2097 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
2098 *
2099 * Return:
2100 *      -EINVAL:  NOT FOUND
2101 *      0..csrow = Chip-Select Row
2102 */
2103static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
2104{
2105        struct mem_ctl_info *mci;
2106        struct amd64_pvt *pvt;
2107        u64 cs_base, cs_mask;
2108        int cs_found = -EINVAL;
2109        int csrow;
2110
2111        mci = edac_mc_find(nid);
2112        if (!mci)
2113                return cs_found;
2114
2115        pvt = mci->pvt_info;
2116
2117        edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
2118
2119        for_each_chip_select(csrow, dct, pvt) {
2120                if (!csrow_enabled(csrow, dct, pvt))
2121                        continue;
2122
2123                get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
2124
2125                edac_dbg(1, "    CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2126                         csrow, cs_base, cs_mask);
2127
2128                cs_mask = ~cs_mask;
2129
2130                edac_dbg(1, "    (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2131                         (in_addr & cs_mask), (cs_base & cs_mask));
2132
2133                if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
2134                        if (pvt->fam == 0x15 && pvt->model >= 0x30) {
2135                                cs_found =  csrow;
2136                                break;
2137                        }
2138                        cs_found = f10_process_possible_spare(pvt, dct, csrow);
2139
2140                        edac_dbg(1, " MATCH csrow=%d\n", cs_found);
2141                        break;
2142                }
2143        }
2144        return cs_found;
2145}
2146
2147/*
2148 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
2149 * swapped with a region located at the bottom of memory so that the GPU can use
2150 * the interleaved region and thus two channels.
2151 */
2152static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
2153{
2154        u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
2155
2156        if (pvt->fam == 0x10) {
2157                /* only revC3 and revE have that feature */
2158                if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
2159                        return sys_addr;
2160        }
2161
2162        amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
2163
2164        if (!(swap_reg & 0x1))
2165                return sys_addr;
2166
2167        swap_base       = (swap_reg >> 3) & 0x7f;
2168        swap_limit      = (swap_reg >> 11) & 0x7f;
2169        rgn_size        = (swap_reg >> 20) & 0x7f;
2170        tmp_addr        = sys_addr >> 27;
2171
2172        if (!(sys_addr >> 34) &&
2173            (((tmp_addr >= swap_base) &&
2174             (tmp_addr <= swap_limit)) ||
2175             (tmp_addr < rgn_size)))
2176                return sys_addr ^ (u64)swap_base << 27;
2177
2178        return sys_addr;
2179}
2180
2181/* For a given @dram_range, check if @sys_addr falls within it. */
2182static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2183                                  u64 sys_addr, int *chan_sel)
2184{
2185        int cs_found = -EINVAL;
2186        u64 chan_addr;
2187        u32 dct_sel_base;
2188        u8 channel;
2189        bool high_range = false;
2190
2191        u8 node_id    = dram_dst_node(pvt, range);
2192        u8 intlv_en   = dram_intlv_en(pvt, range);
2193        u32 intlv_sel = dram_intlv_sel(pvt, range);
2194
2195        edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2196                 range, sys_addr, get_dram_limit(pvt, range));
2197
2198        if (dhar_valid(pvt) &&
2199            dhar_base(pvt) <= sys_addr &&
2200            sys_addr < BIT_64(32)) {
2201                amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2202                            sys_addr);
2203                return -EINVAL;
2204        }
2205
2206        if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
2207                return -EINVAL;
2208
2209        sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
2210
2211        dct_sel_base = dct_sel_baseaddr(pvt);
2212
2213        /*
2214         * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
2215         * select between DCT0 and DCT1.
2216         */
2217        if (dct_high_range_enabled(pvt) &&
2218           !dct_ganging_enabled(pvt) &&
2219           ((sys_addr >> 27) >= (dct_sel_base >> 11)))
2220                high_range = true;
2221
2222        channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
2223
2224        chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
2225                                          high_range, dct_sel_base);
2226
2227        /* Remove node interleaving, see F1x120 */
2228        if (intlv_en)
2229                chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2230                            (chan_addr & 0xfff);
2231
2232        /* remove channel interleave */
2233        if (dct_interleave_enabled(pvt) &&
2234           !dct_high_range_enabled(pvt) &&
2235           !dct_ganging_enabled(pvt)) {
2236
2237                if (dct_sel_interleave_addr(pvt) != 1) {
2238                        if (dct_sel_interleave_addr(pvt) == 0x3)
2239                                /* hash 9 */
2240                                chan_addr = ((chan_addr >> 10) << 9) |
2241                                             (chan_addr & 0x1ff);
2242                        else
2243                                /* A[6] or hash 6 */
2244                                chan_addr = ((chan_addr >> 7) << 6) |
2245                                             (chan_addr & 0x3f);
2246                } else
2247                        /* A[12] */
2248                        chan_addr = ((chan_addr >> 13) << 12) |
2249                                     (chan_addr & 0xfff);
2250        }
2251
2252        edac_dbg(1, "   Normalized DCT addr: 0x%llx\n", chan_addr);
2253
2254        cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
2255
2256        if (cs_found >= 0)
2257                *chan_sel = channel;
2258
2259        return cs_found;
2260}
2261
2262static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2263                                        u64 sys_addr, int *chan_sel)
2264{
2265        int cs_found = -EINVAL;
2266        int num_dcts_intlv = 0;
2267        u64 chan_addr, chan_offset;
2268        u64 dct_base, dct_limit;
2269        u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
2270        u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
2271
2272        u64 dhar_offset         = f10_dhar_offset(pvt);
2273        u8 intlv_addr           = dct_sel_interleave_addr(pvt);
2274        u8 node_id              = dram_dst_node(pvt, range);
2275        u8 intlv_en             = dram_intlv_en(pvt, range);
2276
2277        amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
2278        amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
2279
2280        dct_offset_en           = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
2281        dct_sel                 = (u8) ((dct_cont_base_reg >> 4) & 0x7);
2282
2283        edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2284                 range, sys_addr, get_dram_limit(pvt, range));
2285
2286        if (!(get_dram_base(pvt, range)  <= sys_addr) &&
2287            !(get_dram_limit(pvt, range) >= sys_addr))
2288                return -EINVAL;
2289
2290        if (dhar_valid(pvt) &&
2291            dhar_base(pvt) <= sys_addr &&
2292            sys_addr < BIT_64(32)) {
2293                amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2294                            sys_addr);
2295                return -EINVAL;
2296        }
2297
2298        /* Verify sys_addr is within DCT Range. */
2299        dct_base = (u64) dct_sel_baseaddr(pvt);
2300        dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
2301
2302        if (!(dct_cont_base_reg & BIT(0)) &&
2303            !(dct_base <= (sys_addr >> 27) &&
2304              dct_limit >= (sys_addr >> 27)))
2305                return -EINVAL;
2306
2307        /* Verify number of dct's that participate in channel interleaving. */
2308        num_dcts_intlv = (int) hweight8(intlv_en);
2309
2310        if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
2311                return -EINVAL;
2312
2313        if (pvt->model >= 0x60)
2314                channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2315        else
2316                channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
2317                                                     num_dcts_intlv, dct_sel);
2318
2319        /* Verify we stay within the MAX number of channels allowed */
2320        if (channel > 3)
2321                return -EINVAL;
2322
2323        leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
2324
2325        /* Get normalized DCT addr */
2326        if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
2327                chan_offset = dhar_offset;
2328        else
2329                chan_offset = dct_base << 27;
2330
2331        chan_addr = sys_addr - chan_offset;
2332
2333        /* remove channel interleave */
2334        if (num_dcts_intlv == 2) {
2335                if (intlv_addr == 0x4)
2336                        chan_addr = ((chan_addr >> 9) << 8) |
2337                                                (chan_addr & 0xff);
2338                else if (intlv_addr == 0x5)
2339                        chan_addr = ((chan_addr >> 10) << 9) |
2340                                                (chan_addr & 0x1ff);
2341                else
2342                        return -EINVAL;
2343
2344        } else if (num_dcts_intlv == 4) {
2345                if (intlv_addr == 0x4)
2346                        chan_addr = ((chan_addr >> 10) << 8) |
2347                                                        (chan_addr & 0xff);
2348                else if (intlv_addr == 0x5)
2349                        chan_addr = ((chan_addr >> 11) << 9) |
2350                                                        (chan_addr & 0x1ff);
2351                else
2352                        return -EINVAL;
2353        }
2354
2355        if (dct_offset_en) {
2356                amd64_read_pci_cfg(pvt->F1,
2357                                   DRAM_CONT_HIGH_OFF + (int) channel * 4,
2358                                   &tmp);
2359                chan_addr +=  (u64) ((tmp >> 11) & 0xfff) << 27;
2360        }
2361
2362        f15h_select_dct(pvt, channel);
2363
2364        edac_dbg(1, "   Normalized DCT addr: 0x%llx\n", chan_addr);
2365
2366        /*
2367         * Find Chip select:
2368         * if channel = 3, then alias it to 1. This is because, in F15 M30h,
2369         * there is support for 4 DCT's, but only 2 are currently functional.
2370         * They are DCT0 and DCT3. But we have read all registers of DCT3 into
2371         * pvt->csels[1]. So we need to use '1' here to get correct info.
2372         * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
2373         */
2374        alias_channel =  (channel == 3) ? 1 : channel;
2375
2376        cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2377
2378        if (cs_found >= 0)
2379                *chan_sel = alias_channel;
2380
2381        return cs_found;
2382}
2383
2384static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2385                                        u64 sys_addr,
2386                                        int *chan_sel)
2387{
2388        int cs_found = -EINVAL;
2389        unsigned range;
2390
2391        for (range = 0; range < DRAM_RANGES; range++) {
2392                if (!dram_rw(pvt, range))
2393                        continue;
2394
2395                if (pvt->fam == 0x15 && pvt->model >= 0x30)
2396                        cs_found = f15_m30h_match_to_this_node(pvt, range,
2397                                                               sys_addr,
2398                                                               chan_sel);
2399
2400                else if ((get_dram_base(pvt, range)  <= sys_addr) &&
2401                         (get_dram_limit(pvt, range) >= sys_addr)) {
2402                        cs_found = f1x_match_to_this_node(pvt, range,
2403                                                          sys_addr, chan_sel);
2404                        if (cs_found >= 0)
2405                                break;
2406                }
2407        }
2408        return cs_found;
2409}
2410
2411/*
2412 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2413 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2414 *
2415 * The @sys_addr is usually an error address received from the hardware
2416 * (MCX_ADDR).
2417 */
2418static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2419                                     struct err_info *err)
2420{
2421        struct amd64_pvt *pvt = mci->pvt_info;
2422
2423        error_address_to_page_and_offset(sys_addr, err);
2424
2425        err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2426        if (err->csrow < 0) {
2427                err->err_code = ERR_CSROW;
2428                return;
2429        }
2430
2431        /*
2432         * We need the syndromes for channel detection only when we're
2433         * ganged. Otherwise @chan should already contain the channel at
2434         * this point.
2435         */
2436        if (dct_ganging_enabled(pvt))
2437                err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2438}
2439
2440/*
2441 * debug routine to display the memory sizes of all logical DIMMs and its
2442 * CSROWs
2443 */
2444static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2445{
2446        int dimm, size0, size1;
2447        u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2448        u32 dbam  = ctrl ? pvt->dbam1 : pvt->dbam0;
2449
2450        if (pvt->fam == 0xf) {
2451                /* K8 families < revF not supported yet */
2452               if (pvt->ext_model < K8_REV_F)
2453                        return;
2454               else
2455                       WARN_ON(ctrl != 0);
2456        }
2457
2458        if (pvt->fam == 0x10) {
2459                dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2460                                                           : pvt->dbam0;
2461                dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2462                                 pvt->csels[1].csbases :
2463                                 pvt->csels[0].csbases;
2464        } else if (ctrl) {
2465                dbam = pvt->dbam0;
2466                dcsb = pvt->csels[1].csbases;
2467        }
2468        edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2469                 ctrl, dbam);
2470
2471        edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2472
2473        /* Dump memory sizes for DIMM and its CSROWs */
2474        for (dimm = 0; dimm < 4; dimm++) {
2475
2476                size0 = 0;
2477                if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2478                        /*
2479                         * For F15m60h, we need multiplier for LRDIMM cs_size
2480                         * calculation. We pass dimm value to the dbam_to_cs
2481                         * mapper so we can find the multiplier from the
2482                         * corresponding DCSM.
2483                         */
2484                        size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2485                                                     DBAM_DIMM(dimm, dbam),
2486                                                     dimm);
2487
2488                size1 = 0;
2489                if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2490                        size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2491                                                     DBAM_DIMM(dimm, dbam),
2492                                                     dimm);
2493
2494                amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2495                                dimm * 2,     size0,
2496                                dimm * 2 + 1, size1);
2497        }
2498}
2499
2500static struct amd64_family_type family_types[] = {
2501        [K8_CPUS] = {
2502                .ctl_name = "K8",
2503                .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2504                .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2505                .max_mcs = 2,
2506                .ops = {
2507                        .early_channel_count    = k8_early_channel_count,
2508                        .map_sysaddr_to_csrow   = k8_map_sysaddr_to_csrow,
2509                        .dbam_to_cs             = k8_dbam_to_chip_select,
2510                }
2511        },
2512        [F10_CPUS] = {
2513                .ctl_name = "F10h",
2514                .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2515                .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2516                .max_mcs = 2,
2517                .ops = {
2518                        .early_channel_count    = f1x_early_channel_count,
2519                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
2520                        .dbam_to_cs             = f10_dbam_to_chip_select,
2521                }
2522        },
2523        [F15_CPUS] = {
2524                .ctl_name = "F15h",
2525                .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2526                .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2527                .max_mcs = 2,
2528                .ops = {
2529                        .early_channel_count    = f1x_early_channel_count,
2530                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
2531                        .dbam_to_cs             = f15_dbam_to_chip_select,
2532                }
2533        },
2534        [F15_M30H_CPUS] = {
2535                .ctl_name = "F15h_M30h",
2536                .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2537                .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2538                .max_mcs = 2,
2539                .ops = {
2540                        .early_channel_count    = f1x_early_channel_count,
2541                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
2542                        .dbam_to_cs             = f16_dbam_to_chip_select,
2543                }
2544        },
2545        [F15_M60H_CPUS] = {
2546                .ctl_name = "F15h_M60h",
2547                .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2548                .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2549                .max_mcs = 2,
2550                .ops = {
2551                        .early_channel_count    = f1x_early_channel_count,
2552                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
2553                        .dbam_to_cs             = f15_m60h_dbam_to_chip_select,
2554                }
2555        },
2556        [F16_CPUS] = {
2557                .ctl_name = "F16h",
2558                .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2559                .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2560                .max_mcs = 2,
2561                .ops = {
2562                        .early_channel_count    = f1x_early_channel_count,
2563                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
2564                        .dbam_to_cs             = f16_dbam_to_chip_select,
2565                }
2566        },
2567        [F16_M30H_CPUS] = {
2568                .ctl_name = "F16h_M30h",
2569                .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2570                .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2571                .max_mcs = 2,
2572                .ops = {
2573                        .early_channel_count    = f1x_early_channel_count,
2574                        .map_sysaddr_to_csrow   = f1x_map_sysaddr_to_csrow,
2575                        .dbam_to_cs             = f16_dbam_to_chip_select,
2576                }
2577        },
2578        [F17_CPUS] = {
2579                .ctl_name = "F17h",
2580                .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2581                .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2582                .max_mcs = 2,
2583                .ops = {
2584                        .early_channel_count    = f17_early_channel_count,
2585                        .dbam_to_cs             = f17_addr_mask_to_cs_size,
2586                }
2587        },
2588        [F17_M10H_CPUS] = {
2589                .ctl_name = "F17h_M10h",
2590                .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2591                .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
2592                .max_mcs = 2,
2593                .ops = {
2594                        .early_channel_count    = f17_early_channel_count,
2595                        .dbam_to_cs             = f17_addr_mask_to_cs_size,
2596                }
2597        },
2598        [F17_M30H_CPUS] = {
2599                .ctl_name = "F17h_M30h",
2600                .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2601                .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
2602                .max_mcs = 8,
2603                .ops = {
2604                        .early_channel_count    = f17_early_channel_count,
2605                        .dbam_to_cs             = f17_addr_mask_to_cs_size,
2606                }
2607        },
2608        [F17_M60H_CPUS] = {
2609                .ctl_name = "F17h_M60h",
2610                .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0,
2611                .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6,
2612                .max_mcs = 2,
2613                .ops = {
2614                        .early_channel_count    = f17_early_channel_count,
2615                        .dbam_to_cs             = f17_addr_mask_to_cs_size,
2616                }
2617        },
2618        [F17_M70H_CPUS] = {
2619                .ctl_name = "F17h_M70h",
2620                .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
2621                .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6,
2622                .max_mcs = 2,
2623                .ops = {
2624                        .early_channel_count    = f17_early_channel_count,
2625                        .dbam_to_cs             = f17_addr_mask_to_cs_size,
2626                }
2627        },
2628        [F19_CPUS] = {
2629                .ctl_name = "F19h",
2630                .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0,
2631                .f6_id = PCI_DEVICE_ID_AMD_19H_DF_F6,
2632                .max_mcs = 8,
2633                .ops = {
2634                        .early_channel_count    = f17_early_channel_count,
2635                        .dbam_to_cs             = f17_addr_mask_to_cs_size,
2636                }
2637        },
2638};
2639
2640/*
2641 * These are tables of eigenvectors (one per line) which can be used for the
2642 * construction of the syndrome tables. The modified syndrome search algorithm
2643 * uses those to find the symbol in error and thus the DIMM.
2644 *
2645 * Algorithm courtesy of Ross LaFetra from AMD.
2646 */
2647static const u16 x4_vectors[] = {
2648        0x2f57, 0x1afe, 0x66cc, 0xdd88,
2649        0x11eb, 0x3396, 0x7f4c, 0xeac8,
2650        0x0001, 0x0002, 0x0004, 0x0008,
2651        0x1013, 0x3032, 0x4044, 0x8088,
2652        0x106b, 0x30d6, 0x70fc, 0xe0a8,
2653        0x4857, 0xc4fe, 0x13cc, 0x3288,
2654        0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2655        0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2656        0x15c1, 0x2a42, 0x89ac, 0x4758,
2657        0x2b03, 0x1602, 0x4f0c, 0xca08,
2658        0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2659        0x8ba7, 0x465e, 0x244c, 0x1cc8,
2660        0x2b87, 0x164e, 0x642c, 0xdc18,
2661        0x40b9, 0x80de, 0x1094, 0x20e8,
2662        0x27db, 0x1eb6, 0x9dac, 0x7b58,
2663        0x11c1, 0x2242, 0x84ac, 0x4c58,
2664        0x1be5, 0x2d7a, 0x5e34, 0xa718,
2665        0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2666        0x4c97, 0xc87e, 0x11fc, 0x33a8,
2667        0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2668        0x16b3, 0x3d62, 0x4f34, 0x8518,
2669        0x1e2f, 0x391a, 0x5cac, 0xf858,
2670        0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2671        0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2672        0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2673        0x4397, 0xc27e, 0x17fc, 0x3ea8,
2674        0x1617, 0x3d3e, 0x6464, 0xb8b8,
2675        0x23ff, 0x12aa, 0xab6c, 0x56d8,
2676        0x2dfb, 0x1ba6, 0x913c, 0x7328,
2677        0x185d, 0x2ca6, 0x7914, 0x9e28,
2678        0x171b, 0x3e36, 0x7d7c, 0xebe8,
2679        0x4199, 0x82ee, 0x19f4, 0x2e58,
2680        0x4807, 0xc40e, 0x130c, 0x3208,
2681        0x1905, 0x2e0a, 0x5804, 0xac08,
2682        0x213f, 0x132a, 0xadfc, 0x5ba8,
2683        0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2684};
2685
2686static const u16 x8_vectors[] = {
2687        0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2688        0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2689        0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2690        0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2691        0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2692        0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2693        0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2694        0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2695        0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2696        0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2697        0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2698        0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2699        0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2700        0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2701        0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2702        0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2703        0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2704        0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2705        0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2706};
2707
2708static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2709                           unsigned v_dim)
2710{
2711        unsigned int i, err_sym;
2712
2713        for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2714                u16 s = syndrome;
2715                unsigned v_idx =  err_sym * v_dim;
2716                unsigned v_end = (err_sym + 1) * v_dim;
2717
2718                /* walk over all 16 bits of the syndrome */
2719                for (i = 1; i < (1U << 16); i <<= 1) {
2720
2721                        /* if bit is set in that eigenvector... */
2722                        if (v_idx < v_end && vectors[v_idx] & i) {
2723                                u16 ev_comp = vectors[v_idx++];
2724
2725                                /* ... and bit set in the modified syndrome, */
2726                                if (s & i) {
2727                                        /* remove it. */
2728                                        s ^= ev_comp;
2729
2730                                        if (!s)
2731                                                return err_sym;
2732                                }
2733
2734                        } else if (s & i)
2735                                /* can't get to zero, move to next symbol */
2736                                break;
2737                }
2738        }
2739
2740        edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2741        return -1;
2742}
2743
2744static int map_err_sym_to_channel(int err_sym, int sym_size)
2745{
2746        if (sym_size == 4)
2747                switch (err_sym) {
2748                case 0x20:
2749                case 0x21:
2750                        return 0;
2751                case 0x22:
2752                case 0x23:
2753                        return 1;
2754                default:
2755                        return err_sym >> 4;
2756                }
2757        /* x8 symbols */
2758        else
2759                switch (err_sym) {
2760                /* imaginary bits not in a DIMM */
2761                case 0x10:
2762                        WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2763                                          err_sym);
2764                        return -1;
2765                case 0x11:
2766                        return 0;
2767                case 0x12:
2768                        return 1;
2769                default:
2770                        return err_sym >> 3;
2771                }
2772        return -1;
2773}
2774
2775static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2776{
2777        struct amd64_pvt *pvt = mci->pvt_info;
2778        int err_sym = -1;
2779
2780        if (pvt->ecc_sym_sz == 8)
2781                err_sym = decode_syndrome(syndrome, x8_vectors,
2782                                          ARRAY_SIZE(x8_vectors),
2783                                          pvt->ecc_sym_sz);
2784        else if (pvt->ecc_sym_sz == 4)
2785                err_sym = decode_syndrome(syndrome, x4_vectors,
2786                                          ARRAY_SIZE(x4_vectors),
2787                                          pvt->ecc_sym_sz);
2788        else {
2789                amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2790                return err_sym;
2791        }
2792
2793        return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2794}
2795
2796static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2797                            u8 ecc_type)
2798{
2799        enum hw_event_mc_err_type err_type;
2800        const char *string;
2801
2802        if (ecc_type == 2)
2803                err_type = HW_EVENT_ERR_CORRECTED;
2804        else if (ecc_type == 1)
2805                err_type = HW_EVENT_ERR_UNCORRECTED;
2806        else if (ecc_type == 3)
2807                err_type = HW_EVENT_ERR_DEFERRED;
2808        else {
2809                WARN(1, "Something is rotten in the state of Denmark.\n");
2810                return;
2811        }
2812
2813        switch (err->err_code) {
2814        case DECODE_OK:
2815                string = "";
2816                break;
2817        case ERR_NODE:
2818                string = "Failed to map error addr to a node";
2819                break;
2820        case ERR_CSROW:
2821                string = "Failed to map error addr to a csrow";
2822                break;
2823        case ERR_CHANNEL:
2824                string = "Unknown syndrome - possible error reporting race";
2825                break;
2826        case ERR_SYND:
2827                string = "MCA_SYND not valid - unknown syndrome and csrow";
2828                break;
2829        case ERR_NORM_ADDR:
2830                string = "Cannot decode normalized address";
2831                break;
2832        default:
2833                string = "WTF error";
2834                break;
2835        }
2836
2837        edac_mc_handle_error(err_type, mci, 1,
2838                             err->page, err->offset, err->syndrome,
2839                             err->csrow, err->channel, -1,
2840                             string, "");
2841}
2842
2843static inline void decode_bus_error(int node_id, struct mce *m)
2844{
2845        struct mem_ctl_info *mci;
2846        struct amd64_pvt *pvt;
2847        u8 ecc_type = (m->status >> 45) & 0x3;
2848        u8 xec = XEC(m->status, 0x1f);
2849        u16 ec = EC(m->status);
2850        u64 sys_addr;
2851        struct err_info err;
2852
2853        mci = edac_mc_find(node_id);
2854        if (!mci)
2855                return;
2856
2857        pvt = mci->pvt_info;
2858
2859        /* Bail out early if this was an 'observed' error */
2860        if (PP(ec) == NBSL_PP_OBS)
2861                return;
2862
2863        /* Do only ECC errors */
2864        if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2865                return;
2866
2867        memset(&err, 0, sizeof(err));
2868
2869        sys_addr = get_error_address(pvt, m);
2870
2871        if (ecc_type == 2)
2872                err.syndrome = extract_syndrome(m->status);
2873
2874        pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2875
2876        __log_ecc_error(mci, &err, ecc_type);
2877}
2878
2879/*
2880 * To find the UMC channel represented by this bank we need to match on its
2881 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
2882 * IPID.
2883 *
2884 * Currently, we can derive the channel number by looking at the 6th nibble in
2885 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
2886 * number.
2887 */
2888static int find_umc_channel(struct mce *m)
2889{
2890        return (m->ipid & GENMASK(31, 0)) >> 20;
2891}
2892
2893static void decode_umc_error(int node_id, struct mce *m)
2894{
2895        u8 ecc_type = (m->status >> 45) & 0x3;
2896        struct mem_ctl_info *mci;
2897        struct amd64_pvt *pvt;
2898        struct err_info err;
2899        u64 sys_addr;
2900
2901        mci = edac_mc_find(node_id);
2902        if (!mci)
2903                return;
2904
2905        pvt = mci->pvt_info;
2906
2907        memset(&err, 0, sizeof(err));
2908
2909        if (m->status & MCI_STATUS_DEFERRED)
2910                ecc_type = 3;
2911
2912        err.channel = find_umc_channel(m);
2913
2914        if (!(m->status & MCI_STATUS_SYNDV)) {
2915                err.err_code = ERR_SYND;
2916                goto log_error;
2917        }
2918
2919        if (ecc_type == 2) {
2920                u8 length = (m->synd >> 18) & 0x3f;
2921
2922                if (length)
2923                        err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2924                else
2925                        err.err_code = ERR_CHANNEL;
2926        }
2927
2928        err.csrow = m->synd & 0x7;
2929
2930        if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2931                err.err_code = ERR_NORM_ADDR;
2932                goto log_error;
2933        }
2934
2935        error_address_to_page_and_offset(sys_addr, &err);
2936
2937log_error:
2938        __log_ecc_error(mci, &err, ecc_type);
2939}
2940
2941/*
2942 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2943 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
2944 * Reserve F0 and F6 on systems with a UMC.
2945 */
2946static int
2947reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2948{
2949        if (pvt->umc) {
2950                pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2951                if (!pvt->F0) {
2952                        edac_dbg(1, "F0 not found, device 0x%x\n", pci_id1);
2953                        return -ENODEV;
2954                }
2955
2956                pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2957                if (!pvt->F6) {
2958                        pci_dev_put(pvt->F0);
2959                        pvt->F0 = NULL;
2960
2961                        edac_dbg(1, "F6 not found: device 0x%x\n", pci_id2);
2962                        return -ENODEV;
2963                }
2964
2965                if (!pci_ctl_dev)
2966                        pci_ctl_dev = &pvt->F0->dev;
2967
2968                edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2969                edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2970                edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2971
2972                return 0;
2973        }
2974
2975        /* Reserve the ADDRESS MAP Device */
2976        pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2977        if (!pvt->F1) {
2978                edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
2979                return -ENODEV;
2980        }
2981
2982        /* Reserve the DCT Device */
2983        pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2984        if (!pvt->F2) {
2985                pci_dev_put(pvt->F1);
2986                pvt->F1 = NULL;
2987
2988                edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
2989                return -ENODEV;
2990        }
2991
2992        if (!pci_ctl_dev)
2993                pci_ctl_dev = &pvt->F2->dev;
2994
2995        edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2996        edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2997        edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2998
2999        return 0;
3000}
3001
3002static void free_mc_sibling_devs(struct amd64_pvt *pvt)
3003{
3004        if (pvt->umc) {
3005                pci_dev_put(pvt->F0);
3006                pci_dev_put(pvt->F6);
3007        } else {
3008                pci_dev_put(pvt->F1);
3009                pci_dev_put(pvt->F2);
3010        }
3011}
3012
3013static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3014{
3015        pvt->ecc_sym_sz = 4;
3016
3017        if (pvt->umc) {
3018                u8 i;
3019
3020                for_each_umc(i) {
3021                        /* Check enabled channels only: */
3022                        if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3023                                if (pvt->umc[i].ecc_ctrl & BIT(9)) {
3024                                        pvt->ecc_sym_sz = 16;
3025                                        return;
3026                                } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
3027                                        pvt->ecc_sym_sz = 8;
3028                                        return;
3029                                }
3030                        }
3031                }
3032        } else if (pvt->fam >= 0x10) {
3033                u32 tmp;
3034
3035                amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3036                /* F16h has only DCT0, so no need to read dbam1. */
3037                if (pvt->fam != 0x16)
3038                        amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3039
3040                /* F10h, revD and later can do x8 ECC too. */
3041                if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3042                        pvt->ecc_sym_sz = 8;
3043        }
3044}
3045
3046/*
3047 * Retrieve the hardware registers of the memory controller.
3048 */
3049static void __read_mc_regs_df(struct amd64_pvt *pvt)
3050{
3051        u8 nid = pvt->mc_node_id;
3052        struct amd64_umc *umc;
3053        u32 i, umc_base;
3054
3055        /* Read registers from each UMC */
3056        for_each_umc(i) {
3057
3058                umc_base = get_umc_base(i);
3059                umc = &pvt->umc[i];
3060
3061                amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
3062                amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3063                amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3064                amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
3065                amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
3066        }
3067}
3068
3069/*
3070 * Retrieve the hardware registers of the memory controller (this includes the
3071 * 'Address Map' and 'Misc' device regs)
3072 */
3073static void read_mc_regs(struct amd64_pvt *pvt)
3074{
3075        unsigned int range;
3076        u64 msr_val;
3077
3078        /*
3079         * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
3080         * those are Read-As-Zero.
3081         */
3082        rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
3083        edac_dbg(0, "  TOP_MEM:  0x%016llx\n", pvt->top_mem);
3084
3085        /* Check first whether TOP_MEM2 is enabled: */
3086        rdmsrl(MSR_AMD64_SYSCFG, msr_val);
3087        if (msr_val & BIT(21)) {
3088                rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
3089                edac_dbg(0, "  TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
3090        } else {
3091                edac_dbg(0, "  TOP_MEM2 disabled\n");
3092        }
3093
3094        if (pvt->umc) {
3095                __read_mc_regs_df(pvt);
3096                amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
3097
3098                goto skip;
3099        }
3100
3101        amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
3102
3103        read_dram_ctl_register(pvt);
3104
3105        for (range = 0; range < DRAM_RANGES; range++) {
3106                u8 rw;
3107
3108                /* read settings for this DRAM range */
3109                read_dram_base_limit_regs(pvt, range);
3110
3111                rw = dram_rw(pvt, range);
3112                if (!rw)
3113                        continue;
3114
3115                edac_dbg(1, "  DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
3116                         range,
3117                         get_dram_base(pvt, range),
3118                         get_dram_limit(pvt, range));
3119
3120                edac_dbg(1, "   IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
3121                         dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
3122                         (rw & 0x1) ? "R" : "-",
3123                         (rw & 0x2) ? "W" : "-",
3124                         dram_intlv_sel(pvt, range),
3125                         dram_dst_node(pvt, range));
3126        }
3127
3128        amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
3129        amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
3130
3131        amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
3132
3133        amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
3134        amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
3135
3136        if (!dct_ganging_enabled(pvt)) {
3137                amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
3138                amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
3139        }
3140
3141skip:
3142        read_dct_base_mask(pvt);
3143
3144        determine_memory_type(pvt);
3145        edac_dbg(1, "  DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
3146
3147        determine_ecc_sym_sz(pvt);
3148}
3149
3150/*
3151 * NOTE: CPU Revision Dependent code
3152 *
3153 * Input:
3154 *      @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
3155 *      k8 private pointer to -->
3156 *                      DRAM Bank Address mapping register
3157 *                      node_id
3158 *                      DCL register where dual_channel_active is
3159 *
3160 * The DBAM register consists of 4 sets of 4 bits each definitions:
3161 *
3162 * Bits:        CSROWs
3163 * 0-3          CSROWs 0 and 1
3164 * 4-7          CSROWs 2 and 3
3165 * 8-11         CSROWs 4 and 5
3166 * 12-15        CSROWs 6 and 7
3167 *
3168 * Values range from: 0 to 15
3169 * The meaning of the values depends on CPU revision and dual-channel state,
3170 * see relevant BKDG more info.
3171 *
3172 * The memory controller provides for total of only 8 CSROWs in its current
3173 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
3174 * single channel or two (2) DIMMs in dual channel mode.
3175 *
3176 * The following code logic collapses the various tables for CSROW based on CPU
3177 * revision.
3178 *
3179 * Returns:
3180 *      The number of PAGE_SIZE pages on the specified CSROW number it
3181 *      encompasses
3182 *
3183 */
3184static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
3185{
3186        u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
3187        int csrow_nr = csrow_nr_orig;
3188        u32 cs_mode, nr_pages;
3189
3190        if (!pvt->umc) {
3191                csrow_nr >>= 1;
3192                cs_mode = DBAM_DIMM(csrow_nr, dbam);
3193        } else {
3194                cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
3195        }
3196
3197        nr_pages   = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3198        nr_pages <<= 20 - PAGE_SHIFT;
3199
3200        edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3201                    csrow_nr_orig, dct,  cs_mode);
3202        edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3203
3204        return nr_pages;
3205}
3206
3207static int init_csrows_df(struct mem_ctl_info *mci)
3208{
3209        struct amd64_pvt *pvt = mci->pvt_info;
3210        enum edac_type edac_mode = EDAC_NONE;
3211        enum dev_type dev_type = DEV_UNKNOWN;
3212        struct dimm_info *dimm;
3213        int empty = 1;
3214        u8 umc, cs;
3215
3216        if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3217                edac_mode = EDAC_S16ECD16ED;
3218                dev_type = DEV_X16;
3219        } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3220                edac_mode = EDAC_S8ECD8ED;
3221                dev_type = DEV_X8;
3222        } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3223                edac_mode = EDAC_S4ECD4ED;
3224                dev_type = DEV_X4;
3225        } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3226                edac_mode = EDAC_SECDED;
3227        }
3228
3229        for_each_umc(umc) {
3230                for_each_chip_select(cs, umc, pvt) {
3231                        if (!csrow_enabled(cs, umc, pvt))
3232                                continue;
3233
3234                        empty = 0;
3235                        dimm = mci->csrows[cs]->channels[umc]->dimm;
3236
3237                        edac_dbg(1, "MC node: %d, csrow: %d\n",
3238                                        pvt->mc_node_id, cs);
3239
3240                        dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
3241                        dimm->mtype = pvt->dram_type;
3242                        dimm->edac_mode = edac_mode;
3243                        dimm->dtype = dev_type;
3244                        dimm->grain = 64;
3245                }
3246        }
3247
3248        return empty;
3249}
3250
3251/*
3252 * Initialize the array of csrow attribute instances, based on the values
3253 * from pci config hardware registers.
3254 */
3255static int init_csrows(struct mem_ctl_info *mci)
3256{
3257        struct amd64_pvt *pvt = mci->pvt_info;
3258        enum edac_type edac_mode = EDAC_NONE;
3259        struct csrow_info *csrow;
3260        struct dimm_info *dimm;
3261        int i, j, empty = 1;
3262        int nr_pages = 0;
3263        u32 val;
3264
3265        if (pvt->umc)
3266                return init_csrows_df(mci);
3267
3268        amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
3269
3270        pvt->nbcfg = val;
3271
3272        edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3273                 pvt->mc_node_id, val,
3274                 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
3275
3276        /*
3277         * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
3278         */
3279        for_each_chip_select(i, 0, pvt) {
3280                bool row_dct0 = !!csrow_enabled(i, 0, pvt);
3281                bool row_dct1 = false;
3282
3283                if (pvt->fam != 0xf)
3284                        row_dct1 = !!csrow_enabled(i, 1, pvt);
3285
3286                if (!row_dct0 && !row_dct1)
3287                        continue;
3288
3289                csrow = mci->csrows[i];
3290                empty = 0;
3291
3292                edac_dbg(1, "MC node: %d, csrow: %d\n",
3293                            pvt->mc_node_id, i);
3294
3295                if (row_dct0) {
3296                        nr_pages = get_csrow_nr_pages(pvt, 0, i);
3297                        csrow->channels[0]->dimm->nr_pages = nr_pages;
3298                }
3299
3300                /* K8 has only one DCT */
3301                if (pvt->fam != 0xf && row_dct1) {
3302                        int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
3303
3304                        csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
3305                        nr_pages += row_dct1_pages;
3306                }
3307
3308                edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
3309
3310                /* Determine DIMM ECC mode: */
3311                if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
3312                        edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
3313                                        ? EDAC_S4ECD4ED
3314                                        : EDAC_SECDED;
3315                }
3316
3317                for (j = 0; j < pvt->channel_count; j++) {
3318                        dimm = csrow->channels[j]->dimm;
3319                        dimm->mtype = pvt->dram_type;
3320                        dimm->edac_mode = edac_mode;
3321                        dimm->grain = 64;
3322                }
3323        }
3324
3325        return empty;
3326}
3327
3328/* get all cores on this DCT */
3329static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
3330{
3331        int cpu;
3332
3333        for_each_online_cpu(cpu)
3334                if (topology_die_id(cpu) == nid)
3335                        cpumask_set_cpu(cpu, mask);
3336}
3337
3338/* check MCG_CTL on all the cpus on this node */
3339static bool nb_mce_bank_enabled_on_node(u16 nid)
3340{
3341        cpumask_var_t mask;
3342        int cpu, nbe;
3343        bool ret = false;
3344
3345        if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3346                amd64_warn("%s: Error allocating mask\n", __func__);
3347                return false;
3348        }
3349
3350        get_cpus_on_this_dct_cpumask(mask, nid);
3351
3352        rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
3353
3354        for_each_cpu(cpu, mask) {
3355                struct msr *reg = per_cpu_ptr(msrs, cpu);
3356                nbe = reg->l & MSR_MCGCTL_NBE;
3357
3358                edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3359                         cpu, reg->q,
3360                         (nbe ? "enabled" : "disabled"));
3361
3362                if (!nbe)
3363                        goto out;
3364        }
3365        ret = true;
3366
3367out:
3368        free_cpumask_var(mask);
3369        return ret;
3370}
3371
3372static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
3373{
3374        cpumask_var_t cmask;
3375        int cpu;
3376
3377        if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
3378                amd64_warn("%s: error allocating mask\n", __func__);
3379                return -ENOMEM;
3380        }
3381
3382        get_cpus_on_this_dct_cpumask(cmask, nid);
3383
3384        rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3385
3386        for_each_cpu(cpu, cmask) {
3387
3388                struct msr *reg = per_cpu_ptr(msrs, cpu);
3389
3390                if (on) {
3391                        if (reg->l & MSR_MCGCTL_NBE)
3392                                s->flags.nb_mce_enable = 1;
3393
3394                        reg->l |= MSR_MCGCTL_NBE;
3395                } else {
3396                        /*
3397                         * Turn off NB MCE reporting only when it was off before
3398                         */
3399                        if (!s->flags.nb_mce_enable)
3400                                reg->l &= ~MSR_MCGCTL_NBE;
3401                }
3402        }
3403        wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3404
3405        free_cpumask_var(cmask);
3406
3407        return 0;
3408}
3409
3410static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3411                                       struct pci_dev *F3)
3412{
3413        bool ret = true;
3414        u32 value, mask = 0x3;          /* UECC/CECC enable */
3415
3416        if (toggle_ecc_err_reporting(s, nid, ON)) {
3417                amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3418                return false;
3419        }
3420
3421        amd64_read_pci_cfg(F3, NBCTL, &value);
3422
3423        s->old_nbctl   = value & mask;
3424        s->nbctl_valid = true;
3425
3426        value |= mask;
3427        amd64_write_pci_cfg(F3, NBCTL, value);
3428
3429        amd64_read_pci_cfg(F3, NBCFG, &value);
3430
3431        edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3432                 nid, value, !!(value & NBCFG_ECC_ENABLE));
3433
3434        if (!(value & NBCFG_ECC_ENABLE)) {
3435                amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3436
3437                s->flags.nb_ecc_prev = 0;
3438
3439                /* Attempt to turn on DRAM ECC Enable */
3440                value |= NBCFG_ECC_ENABLE;
3441                amd64_write_pci_cfg(F3, NBCFG, value);
3442
3443                amd64_read_pci_cfg(F3, NBCFG, &value);
3444
3445                if (!(value & NBCFG_ECC_ENABLE)) {
3446                        amd64_warn("Hardware rejected DRAM ECC enable,"
3447                                   "check memory DIMM configuration.\n");
3448                        ret = false;
3449                } else {
3450                        amd64_info("Hardware accepted DRAM ECC Enable\n");
3451                }
3452        } else {
3453                s->flags.nb_ecc_prev = 1;
3454        }
3455
3456        edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3457                 nid, value, !!(value & NBCFG_ECC_ENABLE));
3458
3459        return ret;
3460}
3461
3462static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3463                                        struct pci_dev *F3)
3464{
3465        u32 value, mask = 0x3;          /* UECC/CECC enable */
3466
3467        if (!s->nbctl_valid)
3468                return;
3469
3470        amd64_read_pci_cfg(F3, NBCTL, &value);
3471        value &= ~mask;
3472        value |= s->old_nbctl;
3473
3474        amd64_write_pci_cfg(F3, NBCTL, value);
3475
3476        /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3477        if (!s->flags.nb_ecc_prev) {
3478                amd64_read_pci_cfg(F3, NBCFG, &value);
3479                value &= ~NBCFG_ECC_ENABLE;
3480                amd64_write_pci_cfg(F3, NBCFG, value);
3481        }
3482
3483        /* restore the NB Enable MCGCTL bit */
3484        if (toggle_ecc_err_reporting(s, nid, OFF))
3485                amd64_warn("Error restoring NB MCGCTL settings!\n");
3486}
3487
3488static bool ecc_enabled(struct amd64_pvt *pvt)
3489{
3490        u16 nid = pvt->mc_node_id;
3491        bool nb_mce_en = false;
3492        u8 ecc_en = 0, i;
3493        u32 value;
3494
3495        if (boot_cpu_data.x86 >= 0x17) {
3496                u8 umc_en_mask = 0, ecc_en_mask = 0;
3497                struct amd64_umc *umc;
3498
3499                for_each_umc(i) {
3500                        umc = &pvt->umc[i];
3501
3502                        /* Only check enabled UMCs. */
3503                        if (!(umc->sdp_ctrl & UMC_SDP_INIT))
3504                                continue;
3505
3506                        umc_en_mask |= BIT(i);
3507
3508                        if (umc->umc_cap_hi & UMC_ECC_ENABLED)
3509                                ecc_en_mask |= BIT(i);
3510                }
3511
3512                /* Check whether at least one UMC is enabled: */
3513                if (umc_en_mask)
3514                        ecc_en = umc_en_mask == ecc_en_mask;
3515                else
3516                        edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3517
3518                /* Assume UMC MCA banks are enabled. */
3519                nb_mce_en = true;
3520        } else {
3521                amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
3522
3523                ecc_en = !!(value & NBCFG_ECC_ENABLE);
3524
3525                nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3526                if (!nb_mce_en)
3527                        edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3528                                     MSR_IA32_MCG_CTL, nid);
3529        }
3530
3531        edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3532
3533        if (!ecc_en || !nb_mce_en)
3534                return false;
3535        else
3536                return true;
3537}
3538
3539static inline void
3540f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3541{
3542        u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
3543
3544        for_each_umc(i) {
3545                if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3546                        ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3547                        cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3548
3549                        dev_x4  &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3550                        dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
3551                }
3552        }
3553
3554        /* Set chipkill only if ECC is enabled: */
3555        if (ecc_en) {
3556                mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3557
3558                if (!cpk_en)
3559                        return;
3560
3561                if (dev_x4)
3562                        mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3563                else if (dev_x16)
3564                        mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3565                else
3566                        mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
3567        }
3568}
3569
3570static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
3571{
3572        struct amd64_pvt *pvt = mci->pvt_info;
3573
3574        mci->mtype_cap          = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3575        mci->edac_ctl_cap       = EDAC_FLAG_NONE;
3576
3577        if (pvt->umc) {
3578                f17h_determine_edac_ctl_cap(mci, pvt);
3579        } else {
3580                if (pvt->nbcap & NBCAP_SECDED)
3581                        mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3582
3583                if (pvt->nbcap & NBCAP_CHIPKILL)
3584                        mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3585        }
3586
3587        mci->edac_cap           = determine_edac_cap(pvt);
3588        mci->mod_name           = EDAC_MOD_STR;
3589        mci->ctl_name           = fam_type->ctl_name;
3590        mci->dev_name           = pci_name(pvt->F3);
3591        mci->ctl_page_to_phys   = NULL;
3592
3593        /* memory scrubber interface */
3594        mci->set_sdram_scrub_rate = set_scrub_rate;
3595        mci->get_sdram_scrub_rate = get_scrub_rate;
3596}
3597
3598/*
3599 * returns a pointer to the family descriptor on success, NULL otherwise.
3600 */
3601static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3602{
3603        pvt->ext_model  = boot_cpu_data.x86_model >> 4;
3604        pvt->stepping   = boot_cpu_data.x86_stepping;
3605        pvt->model      = boot_cpu_data.x86_model;
3606        pvt->fam        = boot_cpu_data.x86;
3607
3608        switch (pvt->fam) {
3609        case 0xf:
3610                fam_type        = &family_types[K8_CPUS];
3611                pvt->ops        = &family_types[K8_CPUS].ops;
3612                break;
3613
3614        case 0x10:
3615                fam_type        = &family_types[F10_CPUS];
3616                pvt->ops        = &family_types[F10_CPUS].ops;
3617                break;
3618
3619        case 0x15:
3620                if (pvt->model == 0x30) {
3621                        fam_type = &family_types[F15_M30H_CPUS];
3622                        pvt->ops = &family_types[F15_M30H_CPUS].ops;
3623                        break;
3624                } else if (pvt->model == 0x60) {
3625                        fam_type = &family_types[F15_M60H_CPUS];
3626                        pvt->ops = &family_types[F15_M60H_CPUS].ops;
3627                        break;
3628                /* Richland is only client */
3629                } else if (pvt->model == 0x13) {
3630                        return NULL;
3631                } else {
3632                        fam_type        = &family_types[F15_CPUS];
3633                        pvt->ops        = &family_types[F15_CPUS].ops;
3634                }
3635                break;
3636
3637        case 0x16:
3638                if (pvt->model == 0x30) {
3639                        fam_type = &family_types[F16_M30H_CPUS];
3640                        pvt->ops = &family_types[F16_M30H_CPUS].ops;
3641                        break;
3642                }
3643                fam_type        = &family_types[F16_CPUS];
3644                pvt->ops        = &family_types[F16_CPUS].ops;
3645                break;
3646
3647        case 0x17:
3648                if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3649                        fam_type = &family_types[F17_M10H_CPUS];
3650                        pvt->ops = &family_types[F17_M10H_CPUS].ops;
3651                        break;
3652                } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3653                        fam_type = &family_types[F17_M30H_CPUS];
3654                        pvt->ops = &family_types[F17_M30H_CPUS].ops;
3655                        break;
3656                } else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
3657                        fam_type = &family_types[F17_M60H_CPUS];
3658                        pvt->ops = &family_types[F17_M60H_CPUS].ops;
3659                        break;
3660                } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
3661                        fam_type = &family_types[F17_M70H_CPUS];
3662                        pvt->ops = &family_types[F17_M70H_CPUS].ops;
3663                        break;
3664                }
3665                fallthrough;
3666        case 0x18:
3667                fam_type        = &family_types[F17_CPUS];
3668                pvt->ops        = &family_types[F17_CPUS].ops;
3669
3670                if (pvt->fam == 0x18)
3671                        family_types[F17_CPUS].ctl_name = "F18h";
3672                break;
3673
3674        case 0x19:
3675                if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
3676                        fam_type = &family_types[F17_M70H_CPUS];
3677                        pvt->ops = &family_types[F17_M70H_CPUS].ops;
3678                        fam_type->ctl_name = "F19h_M20h";
3679                        break;
3680                }
3681                fam_type        = &family_types[F19_CPUS];
3682                pvt->ops        = &family_types[F19_CPUS].ops;
3683                family_types[F19_CPUS].ctl_name = "F19h";
3684                break;
3685
3686        default:
3687                amd64_err("Unsupported family!\n");
3688                return NULL;
3689        }
3690
3691        return fam_type;
3692}
3693
3694static const struct attribute_group *amd64_edac_attr_groups[] = {
3695#ifdef CONFIG_EDAC_DEBUG
3696        &dbg_group,
3697        &inj_group,
3698#endif
3699        NULL
3700};
3701
3702static int hw_info_get(struct amd64_pvt *pvt)
3703{
3704        u16 pci_id1, pci_id2;
3705        int ret;
3706
3707        if (pvt->fam >= 0x17) {
3708                pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
3709                if (!pvt->umc)
3710                        return -ENOMEM;
3711
3712                pci_id1 = fam_type->f0_id;
3713                pci_id2 = fam_type->f6_id;
3714        } else {
3715                pci_id1 = fam_type->f1_id;
3716                pci_id2 = fam_type->f2_id;
3717        }
3718
3719        ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3720        if (ret)
3721                return ret;
3722
3723        read_mc_regs(pvt);
3724
3725        return 0;
3726}
3727
3728static void hw_info_put(struct amd64_pvt *pvt)
3729{
3730        if (pvt->F0 || pvt->F1)
3731                free_mc_sibling_devs(pvt);
3732
3733        kfree(pvt->umc);
3734}
3735
3736static int init_one_instance(struct amd64_pvt *pvt)
3737{
3738        struct mem_ctl_info *mci = NULL;
3739        struct edac_mc_layer layers[2];
3740        int ret = -EINVAL;
3741
3742        /*
3743         * We need to determine how many memory channels there are. Then use
3744         * that information for calculating the size of the dynamic instance
3745         * tables in the 'mci' structure.
3746         */
3747        pvt->channel_count = pvt->ops->early_channel_count(pvt);
3748        if (pvt->channel_count < 0)
3749                return ret;
3750
3751        ret = -ENOMEM;
3752        layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3753        layers[0].size = pvt->csels[0].b_cnt;
3754        layers[0].is_virt_csrow = true;
3755        layers[1].type = EDAC_MC_LAYER_CHANNEL;
3756
3757        /*
3758         * Always allocate two channels since we can have setups with DIMMs on
3759         * only one channel. Also, this simplifies handling later for the price
3760         * of a couple of KBs tops.
3761         */
3762        layers[1].size = fam_type->max_mcs;
3763        layers[1].is_virt_csrow = false;
3764
3765        mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
3766        if (!mci)
3767                return ret;
3768
3769        mci->pvt_info = pvt;
3770        mci->pdev = &pvt->F3->dev;
3771
3772        setup_mci_misc_attrs(mci);
3773
3774        if (init_csrows(mci))
3775                mci->edac_cap = EDAC_FLAG_NONE;
3776
3777        ret = -ENODEV;
3778        if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3779                edac_dbg(1, "failed edac_mc_add_mc()\n");
3780                edac_mc_free(mci);
3781                return ret;
3782        }
3783
3784        return 0;
3785}
3786
3787static bool instance_has_memory(struct amd64_pvt *pvt)
3788{
3789        bool cs_enabled = false;
3790        int cs = 0, dct = 0;
3791
3792        for (dct = 0; dct < fam_type->max_mcs; dct++) {
3793                for_each_chip_select(cs, dct, pvt)
3794                        cs_enabled |= csrow_enabled(cs, dct, pvt);
3795        }
3796
3797        return cs_enabled;
3798}
3799
3800static int probe_one_instance(unsigned int nid)
3801{
3802        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3803        struct amd64_pvt *pvt = NULL;
3804        struct ecc_settings *s;
3805        int ret;
3806
3807        ret = -ENOMEM;
3808        s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3809        if (!s)
3810                goto err_out;
3811
3812        ecc_stngs[nid] = s;
3813
3814        pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3815        if (!pvt)
3816                goto err_settings;
3817
3818        pvt->mc_node_id = nid;
3819        pvt->F3 = F3;
3820
3821        ret = -ENODEV;
3822        fam_type = per_family_init(pvt);
3823        if (!fam_type)
3824                goto err_enable;
3825
3826        ret = hw_info_get(pvt);
3827        if (ret < 0)
3828                goto err_enable;
3829
3830        ret = 0;
3831        if (!instance_has_memory(pvt)) {
3832                amd64_info("Node %d: No DIMMs detected.\n", nid);
3833                goto err_enable;
3834        }
3835
3836        if (!ecc_enabled(pvt)) {
3837                ret = -ENODEV;
3838
3839                if (!ecc_enable_override)
3840                        goto err_enable;
3841
3842                if (boot_cpu_data.x86 >= 0x17) {
3843                        amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3844                        goto err_enable;
3845                } else
3846                        amd64_warn("Forcing ECC on!\n");
3847
3848                if (!enable_ecc_error_reporting(s, nid, F3))
3849                        goto err_enable;
3850        }
3851
3852        ret = init_one_instance(pvt);
3853        if (ret < 0) {
3854                amd64_err("Error probing instance: %d\n", nid);
3855
3856                if (boot_cpu_data.x86 < 0x17)
3857                        restore_ecc_error_reporting(s, nid, F3);
3858
3859                goto err_enable;
3860        }
3861
3862        amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3863                     (pvt->fam == 0xf ?
3864                                (pvt->ext_model >= K8_REV_F  ? "revF or later "
3865                                                             : "revE or earlier ")
3866                                 : ""), pvt->mc_node_id);
3867
3868        dump_misc_regs(pvt);
3869
3870        return ret;
3871
3872err_enable:
3873        hw_info_put(pvt);
3874        kfree(pvt);
3875
3876err_settings:
3877        kfree(s);
3878        ecc_stngs[nid] = NULL;
3879
3880err_out:
3881        return ret;
3882}
3883
3884static void remove_one_instance(unsigned int nid)
3885{
3886        struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3887        struct ecc_settings *s = ecc_stngs[nid];
3888        struct mem_ctl_info *mci;
3889        struct amd64_pvt *pvt;
3890
3891        /* Remove from EDAC CORE tracking list */
3892        mci = edac_mc_del_mc(&F3->dev);
3893        if (!mci)
3894                return;
3895
3896        pvt = mci->pvt_info;
3897
3898        restore_ecc_error_reporting(s, nid, F3);
3899
3900        kfree(ecc_stngs[nid]);
3901        ecc_stngs[nid] = NULL;
3902
3903        /* Free the EDAC CORE resources */
3904        mci->pvt_info = NULL;
3905
3906        hw_info_put(pvt);
3907        kfree(pvt);
3908        edac_mc_free(mci);
3909}
3910
3911static void setup_pci_device(void)
3912{
3913        if (pci_ctl)
3914                return;
3915
3916        pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
3917        if (!pci_ctl) {
3918                pr_warn("%s(): Unable to create PCI control\n", __func__);
3919                pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3920        }
3921}
3922
3923static const struct x86_cpu_id amd64_cpuids[] = {
3924        X86_MATCH_VENDOR_FAM(AMD,       0x0F, NULL),
3925        X86_MATCH_VENDOR_FAM(AMD,       0x10, NULL),
3926        X86_MATCH_VENDOR_FAM(AMD,       0x15, NULL),
3927        X86_MATCH_VENDOR_FAM(AMD,       0x16, NULL),
3928        X86_MATCH_VENDOR_FAM(AMD,       0x17, NULL),
3929        X86_MATCH_VENDOR_FAM(HYGON,     0x18, NULL),
3930        X86_MATCH_VENDOR_FAM(AMD,       0x19, NULL),
3931        { }
3932};
3933MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3934
3935static int __init amd64_edac_init(void)
3936{
3937        const char *owner;
3938        int err = -ENODEV;
3939        int i;
3940
3941        owner = edac_get_owner();
3942        if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3943                return -EBUSY;
3944
3945        if (!x86_match_cpu(amd64_cpuids))
3946                return -ENODEV;
3947
3948        if (amd_cache_northbridges() < 0)
3949                return -ENODEV;
3950
3951        opstate_init();
3952
3953        err = -ENOMEM;
3954        ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
3955        if (!ecc_stngs)
3956                goto err_free;
3957
3958        msrs = msrs_alloc();
3959        if (!msrs)
3960                goto err_free;
3961
3962        for (i = 0; i < amd_nb_num(); i++) {
3963                err = probe_one_instance(i);
3964                if (err) {
3965                        /* unwind properly */
3966                        while (--i >= 0)
3967                                remove_one_instance(i);
3968
3969                        goto err_pci;
3970                }
3971        }
3972
3973        if (!edac_has_mcs()) {
3974                err = -ENODEV;
3975                goto err_pci;
3976        }
3977
3978        /* register stuff with EDAC MCE */
3979        if (boot_cpu_data.x86 >= 0x17)
3980                amd_register_ecc_decoder(decode_umc_error);
3981        else
3982                amd_register_ecc_decoder(decode_bus_error);
3983
3984        setup_pci_device();
3985
3986#ifdef CONFIG_X86_32
3987        amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3988#endif
3989
3990        printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3991
3992        return 0;
3993
3994err_pci:
3995        pci_ctl_dev = NULL;
3996
3997        msrs_free(msrs);
3998        msrs = NULL;
3999
4000err_free:
4001        kfree(ecc_stngs);
4002        ecc_stngs = NULL;
4003
4004        return err;
4005}
4006
4007static void __exit amd64_edac_exit(void)
4008{
4009        int i;
4010
4011        if (pci_ctl)
4012                edac_pci_release_generic_ctl(pci_ctl);
4013
4014        /* unregister from EDAC MCE */
4015        if (boot_cpu_data.x86 >= 0x17)
4016                amd_unregister_ecc_decoder(decode_umc_error);
4017        else
4018                amd_unregister_ecc_decoder(decode_bus_error);
4019
4020        for (i = 0; i < amd_nb_num(); i++)
4021                remove_one_instance(i);
4022
4023        kfree(ecc_stngs);
4024        ecc_stngs = NULL;
4025
4026        pci_ctl_dev = NULL;
4027
4028        msrs_free(msrs);
4029        msrs = NULL;
4030}
4031
4032module_init(amd64_edac_init);
4033module_exit(amd64_edac_exit);
4034
4035MODULE_LICENSE("GPL");
4036MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
4037                "Dave Peterson, Thayne Harbaugh");
4038MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
4039                EDAC_AMD64_VERSION);
4040
4041module_param(edac_op_state, int, 0444);
4042MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
4043