1
2#include "amd64_edac.h"
3#include <asm/amd_nb.h>
4
5static struct edac_pci_ctl_info *pci_ctl;
6
7
8
9
10
11static int ecc_enable_override;
12module_param(ecc_enable_override, int, 0644);
13
14static struct msr __percpu *msrs;
15
16static struct amd64_family_type *fam_type;
17
18
19static struct ecc_settings **ecc_stngs;
20
21
22static struct device *pci_ctl_dev;
23
24
25
26
27
28
29
30
31static const struct scrubrate {
32 u32 scrubval;
33 u32 bandwidth;
34} scrubrates[] = {
35 { 0x01, 1600000000UL},
36 { 0x02, 800000000UL},
37 { 0x03, 400000000UL},
38 { 0x04, 200000000UL},
39 { 0x05, 100000000UL},
40 { 0x06, 50000000UL},
41 { 0x07, 25000000UL},
42 { 0x08, 12284069UL},
43 { 0x09, 6274509UL},
44 { 0x0A, 3121951UL},
45 { 0x0B, 1560975UL},
46 { 0x0C, 781440UL},
47 { 0x0D, 390720UL},
48 { 0x0E, 195300UL},
49 { 0x0F, 97650UL},
50 { 0x10, 48854UL},
51 { 0x11, 24427UL},
52 { 0x12, 12213UL},
53 { 0x13, 6101UL},
54 { 0x14, 3051UL},
55 { 0x15, 1523UL},
56 { 0x16, 761UL},
57 { 0x00, 0UL},
58};
59
60int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
61 u32 *val, const char *func)
62{
63 int err = 0;
64
65 err = pci_read_config_dword(pdev, offset, val);
66 if (err)
67 amd64_warn("%s: error reading F%dx%03x.\n",
68 func, PCI_FUNC(pdev->devfn), offset);
69
70 return err;
71}
72
73int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
74 u32 val, const char *func)
75{
76 int err = 0;
77
78 err = pci_write_config_dword(pdev, offset, val);
79 if (err)
80 amd64_warn("%s: error writing to F%dx%03x.\n",
81 func, PCI_FUNC(pdev->devfn), offset);
82
83 return err;
84}
85
86
87
88
89static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
90{
91 u32 reg = 0;
92
93 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
94 reg &= (pvt->model == 0x30) ? ~3 : ~1;
95 reg |= dct;
96 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
97}
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
114 int offset, u32 *val)
115{
116 switch (pvt->fam) {
117 case 0xf:
118 if (dct || offset >= 0x100)
119 return -EINVAL;
120 break;
121
122 case 0x10:
123 if (dct) {
124
125
126
127
128
129 if (dct_ganging_enabled(pvt))
130 return 0;
131
132 offset += 0x100;
133 }
134 break;
135
136 case 0x15:
137
138
139
140
141 dct = (dct && pvt->model == 0x30) ? 3 : dct;
142 f15h_select_dct(pvt, dct);
143 break;
144
145 case 0x16:
146 if (dct)
147 return -EINVAL;
148 break;
149
150 default:
151 break;
152 }
153 return amd64_read_pci_cfg(pvt->F2, offset, val);
154}
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
171{
172
173
174
175
176
177 if (scrubval >= 0x5 && scrubval <= 0x14) {
178 scrubval -= 0x5;
179 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
180 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
181 } else {
182 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
183 }
184}
185
186
187
188
189static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
190{
191 u32 scrubval;
192 int i;
193
194
195
196
197
198
199
200
201
202
203 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
204
205
206
207
208 if (scrubrates[i].scrubval < min_rate)
209 continue;
210
211 if (scrubrates[i].bandwidth <= new_bw)
212 break;
213 }
214
215 scrubval = scrubrates[i].scrubval;
216
217 if (pvt->umc) {
218 __f17h_set_scrubval(pvt, scrubval);
219 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
220 f15h_select_dct(pvt, 0);
221 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
222 f15h_select_dct(pvt, 1);
223 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
224 } else {
225 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
226 }
227
228 if (scrubval)
229 return scrubrates[i].bandwidth;
230
231 return 0;
232}
233
234static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
235{
236 struct amd64_pvt *pvt = mci->pvt_info;
237 u32 min_scrubrate = 0x5;
238
239 if (pvt->fam == 0xf)
240 min_scrubrate = 0x0;
241
242 if (pvt->fam == 0x15) {
243
244 if (pvt->model < 0x10)
245 f15h_select_dct(pvt, 0);
246
247 if (pvt->model == 0x60)
248 min_scrubrate = 0x6;
249 }
250 return __set_scrub_rate(pvt, bw, min_scrubrate);
251}
252
253static int get_scrub_rate(struct mem_ctl_info *mci)
254{
255 struct amd64_pvt *pvt = mci->pvt_info;
256 int i, retval = -EINVAL;
257 u32 scrubval = 0;
258
259 if (pvt->umc) {
260 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
261 if (scrubval & BIT(0)) {
262 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
263 scrubval &= 0xF;
264 scrubval += 0x5;
265 } else {
266 scrubval = 0;
267 }
268 } else if (pvt->fam == 0x15) {
269
270 if (pvt->model < 0x10)
271 f15h_select_dct(pvt, 0);
272
273 if (pvt->model == 0x60)
274 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
275 else
276 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
277 } else {
278 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
279 }
280
281 scrubval = scrubval & 0x001F;
282
283 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
284 if (scrubrates[i].scrubval == scrubval) {
285 retval = scrubrates[i].bandwidth;
286 break;
287 }
288 }
289 return retval;
290}
291
292
293
294
295
296static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
297{
298 u64 addr;
299
300
301
302
303
304
305
306 addr = sys_addr & 0x000000ffffffffffull;
307
308 return ((addr >= get_dram_base(pvt, nid)) &&
309 (addr <= get_dram_limit(pvt, nid)));
310}
311
312
313
314
315
316
317
318static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
319 u64 sys_addr)
320{
321 struct amd64_pvt *pvt;
322 u8 node_id;
323 u32 intlv_en, bits;
324
325
326
327
328
329 pvt = mci->pvt_info;
330
331
332
333
334
335
336 intlv_en = dram_intlv_en(pvt, 0);
337
338 if (intlv_en == 0) {
339 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
340 if (base_limit_match(pvt, sys_addr, node_id))
341 goto found;
342 }
343 goto err_no_match;
344 }
345
346 if (unlikely((intlv_en != 0x01) &&
347 (intlv_en != 0x03) &&
348 (intlv_en != 0x07))) {
349 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
350 return NULL;
351 }
352
353 bits = (((u32) sys_addr) >> 12) & intlv_en;
354
355 for (node_id = 0; ; ) {
356 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
357 break;
358
359 if (++node_id >= DRAM_RANGES)
360 goto err_no_match;
361 }
362
363
364 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
365 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
366 "range for node %d with node interleaving enabled.\n",
367 __func__, sys_addr, node_id);
368 return NULL;
369 }
370
371found:
372 return edac_mc_find((int)node_id);
373
374err_no_match:
375 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
376 (unsigned long)sys_addr);
377
378 return NULL;
379}
380
381
382
383
384
385static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
386 u64 *base, u64 *mask)
387{
388 u64 csbase, csmask, base_bits, mask_bits;
389 u8 addr_shift;
390
391 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
392 csbase = pvt->csels[dct].csbases[csrow];
393 csmask = pvt->csels[dct].csmasks[csrow];
394 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
395 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
396 addr_shift = 4;
397
398
399
400
401
402 } else if (pvt->fam == 0x16 ||
403 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
404 csbase = pvt->csels[dct].csbases[csrow];
405 csmask = pvt->csels[dct].csmasks[csrow >> 1];
406
407 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
408 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
409
410 *mask = ~0ULL;
411
412 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
413 (GENMASK_ULL(30, 19) << 8));
414
415 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
416 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
417
418 return;
419 } else {
420 csbase = pvt->csels[dct].csbases[csrow];
421 csmask = pvt->csels[dct].csmasks[csrow >> 1];
422 addr_shift = 8;
423
424 if (pvt->fam == 0x15)
425 base_bits = mask_bits =
426 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
427 else
428 base_bits = mask_bits =
429 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
430 }
431
432 *base = (csbase & base_bits) << addr_shift;
433
434 *mask = ~0ULL;
435
436 *mask &= ~(mask_bits << addr_shift);
437
438 *mask |= (csmask & mask_bits) << addr_shift;
439}
440
441#define for_each_chip_select(i, dct, pvt) \
442 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
443
444#define chip_select_base(i, dct, pvt) \
445 pvt->csels[dct].csbases[i]
446
447#define for_each_chip_select_mask(i, dct, pvt) \
448 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
449
450#define for_each_umc(i) \
451 for (i = 0; i < fam_type->max_mcs; i++)
452
453
454
455
456
457static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
458{
459 struct amd64_pvt *pvt;
460 int csrow;
461 u64 base, mask;
462
463 pvt = mci->pvt_info;
464
465 for_each_chip_select(csrow, 0, pvt) {
466 if (!csrow_enabled(csrow, 0, pvt))
467 continue;
468
469 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
470
471 mask = ~mask;
472
473 if ((input_addr & mask) == (base & mask)) {
474 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
475 (unsigned long)input_addr, csrow,
476 pvt->mc_node_id);
477
478 return csrow;
479 }
480 }
481 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
482 (unsigned long)input_addr, pvt->mc_node_id);
483
484 return -1;
485}
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
504 u64 *hole_offset, u64 *hole_size)
505{
506 struct amd64_pvt *pvt = mci->pvt_info;
507
508
509 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
510 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
511 pvt->ext_model, pvt->mc_node_id);
512 return 1;
513 }
514
515
516 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
517 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
518 return 1;
519 }
520
521 if (!dhar_valid(pvt)) {
522 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
523 pvt->mc_node_id);
524 return 1;
525 }
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545 *hole_base = dhar_base(pvt);
546 *hole_size = (1ULL << 32) - *hole_base;
547
548 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
549 : k8_dhar_offset(pvt);
550
551 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
552 pvt->mc_node_id, (unsigned long)*hole_base,
553 (unsigned long)*hole_offset, (unsigned long)*hole_size);
554
555 return 0;
556}
557
558#ifdef CONFIG_EDAC_DEBUG
559#define EDAC_DCT_ATTR_SHOW(reg) \
560static ssize_t reg##_show(struct device *dev, \
561 struct device_attribute *mattr, char *data) \
562{ \
563 struct mem_ctl_info *mci = to_mci(dev); \
564 struct amd64_pvt *pvt = mci->pvt_info; \
565 \
566 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
567}
568
569EDAC_DCT_ATTR_SHOW(dhar);
570EDAC_DCT_ATTR_SHOW(dbam0);
571EDAC_DCT_ATTR_SHOW(top_mem);
572EDAC_DCT_ATTR_SHOW(top_mem2);
573
574static ssize_t hole_show(struct device *dev, struct device_attribute *mattr,
575 char *data)
576{
577 struct mem_ctl_info *mci = to_mci(dev);
578
579 u64 hole_base = 0;
580 u64 hole_offset = 0;
581 u64 hole_size = 0;
582
583 get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
584
585 return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
586 hole_size);
587}
588
589
590
591
592static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
593static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
594static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
595static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
596static DEVICE_ATTR(dram_hole, S_IRUGO, hole_show, NULL);
597
598static struct attribute *dbg_attrs[] = {
599 &dev_attr_dhar.attr,
600 &dev_attr_dbam.attr,
601 &dev_attr_topmem.attr,
602 &dev_attr_topmem2.attr,
603 &dev_attr_dram_hole.attr,
604 NULL
605};
606
607static const struct attribute_group dbg_group = {
608 .attrs = dbg_attrs,
609};
610
611static ssize_t inject_section_show(struct device *dev,
612 struct device_attribute *mattr, char *buf)
613{
614 struct mem_ctl_info *mci = to_mci(dev);
615 struct amd64_pvt *pvt = mci->pvt_info;
616 return sprintf(buf, "0x%x\n", pvt->injection.section);
617}
618
619
620
621
622
623
624
625static ssize_t inject_section_store(struct device *dev,
626 struct device_attribute *mattr,
627 const char *data, size_t count)
628{
629 struct mem_ctl_info *mci = to_mci(dev);
630 struct amd64_pvt *pvt = mci->pvt_info;
631 unsigned long value;
632 int ret;
633
634 ret = kstrtoul(data, 10, &value);
635 if (ret < 0)
636 return ret;
637
638 if (value > 3) {
639 amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
640 return -EINVAL;
641 }
642
643 pvt->injection.section = (u32) value;
644 return count;
645}
646
647static ssize_t inject_word_show(struct device *dev,
648 struct device_attribute *mattr, char *buf)
649{
650 struct mem_ctl_info *mci = to_mci(dev);
651 struct amd64_pvt *pvt = mci->pvt_info;
652 return sprintf(buf, "0x%x\n", pvt->injection.word);
653}
654
655
656
657
658
659
660
661static ssize_t inject_word_store(struct device *dev,
662 struct device_attribute *mattr,
663 const char *data, size_t count)
664{
665 struct mem_ctl_info *mci = to_mci(dev);
666 struct amd64_pvt *pvt = mci->pvt_info;
667 unsigned long value;
668 int ret;
669
670 ret = kstrtoul(data, 10, &value);
671 if (ret < 0)
672 return ret;
673
674 if (value > 8) {
675 amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
676 return -EINVAL;
677 }
678
679 pvt->injection.word = (u32) value;
680 return count;
681}
682
683static ssize_t inject_ecc_vector_show(struct device *dev,
684 struct device_attribute *mattr,
685 char *buf)
686{
687 struct mem_ctl_info *mci = to_mci(dev);
688 struct amd64_pvt *pvt = mci->pvt_info;
689 return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
690}
691
692
693
694
695
696
697static ssize_t inject_ecc_vector_store(struct device *dev,
698 struct device_attribute *mattr,
699 const char *data, size_t count)
700{
701 struct mem_ctl_info *mci = to_mci(dev);
702 struct amd64_pvt *pvt = mci->pvt_info;
703 unsigned long value;
704 int ret;
705
706 ret = kstrtoul(data, 16, &value);
707 if (ret < 0)
708 return ret;
709
710 if (value & 0xFFFF0000) {
711 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
712 return -EINVAL;
713 }
714
715 pvt->injection.bit_map = (u32) value;
716 return count;
717}
718
719
720
721
722
723static ssize_t inject_read_store(struct device *dev,
724 struct device_attribute *mattr,
725 const char *data, size_t count)
726{
727 struct mem_ctl_info *mci = to_mci(dev);
728 struct amd64_pvt *pvt = mci->pvt_info;
729 unsigned long value;
730 u32 section, word_bits;
731 int ret;
732
733 ret = kstrtoul(data, 10, &value);
734 if (ret < 0)
735 return ret;
736
737
738 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
739
740 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
741
742 word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
743
744
745 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
746
747 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
748
749 return count;
750}
751
752
753
754
755
756static ssize_t inject_write_store(struct device *dev,
757 struct device_attribute *mattr,
758 const char *data, size_t count)
759{
760 struct mem_ctl_info *mci = to_mci(dev);
761 struct amd64_pvt *pvt = mci->pvt_info;
762 u32 section, word_bits, tmp;
763 unsigned long value;
764 int ret;
765
766 ret = kstrtoul(data, 10, &value);
767 if (ret < 0)
768 return ret;
769
770
771 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
772
773 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
774
775 word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
776
777 pr_notice_once("Don't forget to decrease MCE polling interval in\n"
778 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
779 "so that you can get the error report faster.\n");
780
781 on_each_cpu(disable_caches, NULL, 1);
782
783
784 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
785
786 retry:
787
788 amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
789 if (tmp & F10_NB_ARR_ECC_WR_REQ) {
790 cpu_relax();
791 goto retry;
792 }
793
794 on_each_cpu(enable_caches, NULL, 1);
795
796 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
797
798 return count;
799}
800
801
802
803
804
805static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR,
806 inject_section_show, inject_section_store);
807static DEVICE_ATTR(inject_word, S_IRUGO | S_IWUSR,
808 inject_word_show, inject_word_store);
809static DEVICE_ATTR(inject_ecc_vector, S_IRUGO | S_IWUSR,
810 inject_ecc_vector_show, inject_ecc_vector_store);
811static DEVICE_ATTR(inject_write, S_IWUSR,
812 NULL, inject_write_store);
813static DEVICE_ATTR(inject_read, S_IWUSR,
814 NULL, inject_read_store);
815
816static struct attribute *inj_attrs[] = {
817 &dev_attr_inject_section.attr,
818 &dev_attr_inject_word.attr,
819 &dev_attr_inject_ecc_vector.attr,
820 &dev_attr_inject_write.attr,
821 &dev_attr_inject_read.attr,
822 NULL
823};
824
825static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
826{
827 struct device *dev = kobj_to_dev(kobj);
828 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
829 struct amd64_pvt *pvt = mci->pvt_info;
830
831
832 if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
833 return attr->mode;
834
835 return 0;
836}
837
838static const struct attribute_group inj_group = {
839 .attrs = inj_attrs,
840 .is_visible = inj_is_visible,
841};
842#endif
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
874{
875 struct amd64_pvt *pvt = mci->pvt_info;
876 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
877 int ret;
878
879 dram_base = get_dram_base(pvt, pvt->mc_node_id);
880
881 ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
882 if (!ret) {
883 if ((sys_addr >= (1ULL << 32)) &&
884 (sys_addr < ((1ULL << 32) + hole_size))) {
885
886 dram_addr = sys_addr - hole_offset;
887
888 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
889 (unsigned long)sys_addr,
890 (unsigned long)dram_addr);
891
892 return dram_addr;
893 }
894 }
895
896
897
898
899
900
901
902
903
904
905 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
906
907 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
908 (unsigned long)sys_addr, (unsigned long)dram_addr);
909 return dram_addr;
910}
911
912
913
914
915
916
917static int num_node_interleave_bits(unsigned intlv_en)
918{
919 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
920 int n;
921
922 BUG_ON(intlv_en > 7);
923 n = intlv_shift_table[intlv_en];
924 return n;
925}
926
927
928static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
929{
930 struct amd64_pvt *pvt;
931 int intlv_shift;
932 u64 input_addr;
933
934 pvt = mci->pvt_info;
935
936
937
938
939
940 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
941 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
942 (dram_addr & 0xfff);
943
944 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
945 intlv_shift, (unsigned long)dram_addr,
946 (unsigned long)input_addr);
947
948 return input_addr;
949}
950
951
952
953
954
955static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
956{
957 u64 input_addr;
958
959 input_addr =
960 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
961
962 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
963 (unsigned long)sys_addr, (unsigned long)input_addr);
964
965 return input_addr;
966}
967
968
969static inline void error_address_to_page_and_offset(u64 error_address,
970 struct err_info *err)
971{
972 err->page = (u32) (error_address >> PAGE_SHIFT);
973 err->offset = ((u32) error_address) & ~PAGE_MASK;
974}
975
976
977
978
979
980
981
982
983
984static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
985{
986 int csrow;
987
988 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
989
990 if (csrow == -1)
991 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
992 "address 0x%lx\n", (unsigned long)sys_addr);
993 return csrow;
994}
995
996static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
997
998
999
1000
1001
1002static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
1003{
1004 unsigned long edac_cap = EDAC_FLAG_NONE;
1005 u8 bit;
1006
1007 if (pvt->umc) {
1008 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
1009
1010 for_each_umc(i) {
1011 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1012 continue;
1013
1014 umc_en_mask |= BIT(i);
1015
1016
1017 if (pvt->umc[i].umc_cfg & BIT(12))
1018 dimm_ecc_en_mask |= BIT(i);
1019 }
1020
1021 if (umc_en_mask == dimm_ecc_en_mask)
1022 edac_cap = EDAC_FLAG_SECDED;
1023 } else {
1024 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1025 ? 19
1026 : 17;
1027
1028 if (pvt->dclr0 & BIT(bit))
1029 edac_cap = EDAC_FLAG_SECDED;
1030 }
1031
1032 return edac_cap;
1033}
1034
1035static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
1036
1037static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
1038{
1039 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
1040
1041 if (pvt->dram_type == MEM_LRDDR3) {
1042 u32 dcsm = pvt->csels[chan].csmasks[0];
1043
1044
1045
1046
1047
1048 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1049 }
1050
1051 edac_dbg(1, "All DIMMs support ECC:%s\n",
1052 (dclr & BIT(19)) ? "yes" : "no");
1053
1054
1055 edac_dbg(1, " PAR/ERR parity: %s\n",
1056 (dclr & BIT(8)) ? "enabled" : "disabled");
1057
1058 if (pvt->fam == 0x10)
1059 edac_dbg(1, " DCT 128bit mode width: %s\n",
1060 (dclr & BIT(11)) ? "128b" : "64b");
1061
1062 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1063 (dclr & BIT(12)) ? "yes" : "no",
1064 (dclr & BIT(13)) ? "yes" : "no",
1065 (dclr & BIT(14)) ? "yes" : "no",
1066 (dclr & BIT(15)) ? "yes" : "no");
1067}
1068
1069#define CS_EVEN_PRIMARY BIT(0)
1070#define CS_ODD_PRIMARY BIT(1)
1071#define CS_EVEN_SECONDARY BIT(2)
1072#define CS_ODD_SECONDARY BIT(3)
1073
1074#define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1075#define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1076
1077static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
1078{
1079 int cs_mode = 0;
1080
1081 if (csrow_enabled(2 * dimm, ctrl, pvt))
1082 cs_mode |= CS_EVEN_PRIMARY;
1083
1084 if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1085 cs_mode |= CS_ODD_PRIMARY;
1086
1087
1088 if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
1089 cs_mode |= CS_ODD_SECONDARY;
1090
1091 return cs_mode;
1092}
1093
1094static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
1095{
1096 int dimm, size0, size1, cs0, cs1, cs_mode;
1097
1098 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
1099
1100 for (dimm = 0; dimm < 2; dimm++) {
1101 cs0 = dimm * 2;
1102 cs1 = dimm * 2 + 1;
1103
1104 cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
1105
1106 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
1107 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);
1108
1109 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1110 cs0, size0,
1111 cs1, size1);
1112 }
1113}
1114
1115static void __dump_misc_regs_df(struct amd64_pvt *pvt)
1116{
1117 struct amd64_umc *umc;
1118 u32 i, tmp, umc_base;
1119
1120 for_each_umc(i) {
1121 umc_base = get_umc_base(i);
1122 umc = &pvt->umc[i];
1123
1124 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
1125 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
1126 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
1127 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
1128
1129 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
1130 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
1131
1132 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
1133 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
1134 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
1135
1136 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1137 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
1138 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
1139 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1140 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
1141 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1142 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
1143 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1144 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
1145
1146 if (pvt->dram_type == MEM_LRDDR4) {
1147 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
1148 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1149 i, 1 << ((tmp >> 4) & 0x3));
1150 }
1151
1152 debug_display_dimm_sizes_df(pvt, i);
1153 }
1154
1155 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
1156 pvt->dhar, dhar_base(pvt));
1157}
1158
1159
1160static void __dump_misc_regs(struct amd64_pvt *pvt)
1161{
1162 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
1163
1164 edac_dbg(1, " NB two channel DRAM capable: %s\n",
1165 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
1166
1167 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
1168 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
1169 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
1170
1171 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
1172
1173 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
1174
1175 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1176 pvt->dhar, dhar_base(pvt),
1177 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1178 : f10_dhar_offset(pvt));
1179
1180 debug_display_dimm_sizes(pvt, 0);
1181
1182
1183 if (pvt->fam == 0xf)
1184 return;
1185
1186 debug_display_dimm_sizes(pvt, 1);
1187
1188
1189 if (!dct_ganging_enabled(pvt))
1190 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
1191}
1192
1193
1194static void dump_misc_regs(struct amd64_pvt *pvt)
1195{
1196 if (pvt->umc)
1197 __dump_misc_regs_df(pvt);
1198 else
1199 __dump_misc_regs(pvt);
1200
1201 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
1202
1203 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
1204}
1205
1206
1207
1208
1209static void prep_chip_selects(struct amd64_pvt *pvt)
1210{
1211 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
1212 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1213 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
1214 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
1215 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
1216 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
1217 } else if (pvt->fam >= 0x17) {
1218 int umc;
1219
1220 for_each_umc(umc) {
1221 pvt->csels[umc].b_cnt = 4;
1222 pvt->csels[umc].m_cnt = 2;
1223 }
1224
1225 } else {
1226 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1227 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
1228 }
1229}
1230
1231static void read_umc_base_mask(struct amd64_pvt *pvt)
1232{
1233 u32 umc_base_reg, umc_base_reg_sec;
1234 u32 umc_mask_reg, umc_mask_reg_sec;
1235 u32 base_reg, base_reg_sec;
1236 u32 mask_reg, mask_reg_sec;
1237 u32 *base, *base_sec;
1238 u32 *mask, *mask_sec;
1239 int cs, umc;
1240
1241 for_each_umc(umc) {
1242 umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
1243 umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
1244
1245 for_each_chip_select(cs, umc, pvt) {
1246 base = &pvt->csels[umc].csbases[cs];
1247 base_sec = &pvt->csels[umc].csbases_sec[cs];
1248
1249 base_reg = umc_base_reg + (cs * 4);
1250 base_reg_sec = umc_base_reg_sec + (cs * 4);
1251
1252 if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1253 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1254 umc, cs, *base, base_reg);
1255
1256 if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
1257 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1258 umc, cs, *base_sec, base_reg_sec);
1259 }
1260
1261 umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
1262 umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;
1263
1264 for_each_chip_select_mask(cs, umc, pvt) {
1265 mask = &pvt->csels[umc].csmasks[cs];
1266 mask_sec = &pvt->csels[umc].csmasks_sec[cs];
1267
1268 mask_reg = umc_mask_reg + (cs * 4);
1269 mask_reg_sec = umc_mask_reg_sec + (cs * 4);
1270
1271 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1272 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1273 umc, cs, *mask, mask_reg);
1274
1275 if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
1276 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1277 umc, cs, *mask_sec, mask_reg_sec);
1278 }
1279 }
1280}
1281
1282
1283
1284
1285static void read_dct_base_mask(struct amd64_pvt *pvt)
1286{
1287 int cs;
1288
1289 prep_chip_selects(pvt);
1290
1291 if (pvt->umc)
1292 return read_umc_base_mask(pvt);
1293
1294 for_each_chip_select(cs, 0, pvt) {
1295 int reg0 = DCSB0 + (cs * 4);
1296 int reg1 = DCSB1 + (cs * 4);
1297 u32 *base0 = &pvt->csels[0].csbases[cs];
1298 u32 *base1 = &pvt->csels[1].csbases[cs];
1299
1300 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1301 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
1302 cs, *base0, reg0);
1303
1304 if (pvt->fam == 0xf)
1305 continue;
1306
1307 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1308 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
1309 cs, *base1, (pvt->fam == 0x10) ? reg1
1310 : reg0);
1311 }
1312
1313 for_each_chip_select_mask(cs, 0, pvt) {
1314 int reg0 = DCSM0 + (cs * 4);
1315 int reg1 = DCSM1 + (cs * 4);
1316 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1317 u32 *mask1 = &pvt->csels[1].csmasks[cs];
1318
1319 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1320 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1321 cs, *mask0, reg0);
1322
1323 if (pvt->fam == 0xf)
1324 continue;
1325
1326 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1327 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1328 cs, *mask1, (pvt->fam == 0x10) ? reg1
1329 : reg0);
1330 }
1331}
1332
1333static void determine_memory_type(struct amd64_pvt *pvt)
1334{
1335 u32 dram_ctrl, dcsm;
1336
1337 if (pvt->umc) {
1338 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1339 pvt->dram_type = MEM_LRDDR4;
1340 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1341 pvt->dram_type = MEM_RDDR4;
1342 else
1343 pvt->dram_type = MEM_DDR4;
1344 return;
1345 }
1346
1347 switch (pvt->fam) {
1348 case 0xf:
1349 if (pvt->ext_model >= K8_REV_F)
1350 goto ddr3;
1351
1352 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1353 return;
1354
1355 case 0x10:
1356 if (pvt->dchr0 & DDR3_MODE)
1357 goto ddr3;
1358
1359 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1360 return;
1361
1362 case 0x15:
1363 if (pvt->model < 0x60)
1364 goto ddr3;
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1376 dcsm = pvt->csels[0].csmasks[0];
1377
1378 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1379 pvt->dram_type = MEM_DDR4;
1380 else if (pvt->dclr0 & BIT(16))
1381 pvt->dram_type = MEM_DDR3;
1382 else if (dcsm & 0x3)
1383 pvt->dram_type = MEM_LRDDR3;
1384 else
1385 pvt->dram_type = MEM_RDDR3;
1386
1387 return;
1388
1389 case 0x16:
1390 goto ddr3;
1391
1392 default:
1393 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1394 pvt->dram_type = MEM_EMPTY;
1395 }
1396 return;
1397
1398ddr3:
1399 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1400}
1401
1402
1403static int k8_early_channel_count(struct amd64_pvt *pvt)
1404{
1405 int flag;
1406
1407 if (pvt->ext_model >= K8_REV_F)
1408
1409 flag = pvt->dclr0 & WIDTH_128;
1410 else
1411
1412 flag = pvt->dclr0 & REVE_WIDTH_128;
1413
1414
1415 pvt->dclr1 = 0;
1416
1417 return (flag) ? 2 : 1;
1418}
1419
1420
1421static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1422{
1423 u16 mce_nid = topology_die_id(m->extcpu);
1424 struct mem_ctl_info *mci;
1425 u8 start_bit = 1;
1426 u8 end_bit = 47;
1427 u64 addr;
1428
1429 mci = edac_mc_find(mce_nid);
1430 if (!mci)
1431 return 0;
1432
1433 pvt = mci->pvt_info;
1434
1435 if (pvt->fam == 0xf) {
1436 start_bit = 3;
1437 end_bit = 39;
1438 }
1439
1440 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1441
1442
1443
1444
1445 if (pvt->fam == 0x15) {
1446 u64 cc6_base, tmp_addr;
1447 u32 tmp;
1448 u8 intlv_en;
1449
1450 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1451 return addr;
1452
1453
1454 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1455 intlv_en = tmp >> 21 & 0x7;
1456
1457
1458 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1459
1460
1461 cc6_base |= intlv_en ^ 0x7;
1462
1463
1464 cc6_base <<= 24;
1465
1466 if (!intlv_en)
1467 return cc6_base | (addr & GENMASK_ULL(23, 0));
1468
1469 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1470
1471
1472 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1473
1474
1475 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1476
1477
1478 tmp_addr |= addr & GENMASK_ULL(11, 0);
1479
1480 return cc6_base | tmp_addr;
1481 }
1482
1483 return addr;
1484}
1485
1486static struct pci_dev *pci_get_related_function(unsigned int vendor,
1487 unsigned int device,
1488 struct pci_dev *related)
1489{
1490 struct pci_dev *dev = NULL;
1491
1492 while ((dev = pci_get_device(vendor, device, dev))) {
1493 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1494 (dev->bus->number == related->bus->number) &&
1495 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1496 break;
1497 }
1498
1499 return dev;
1500}
1501
1502static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1503{
1504 struct amd_northbridge *nb;
1505 struct pci_dev *f1 = NULL;
1506 unsigned int pci_func;
1507 int off = range << 3;
1508 u32 llim;
1509
1510 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1511 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1512
1513 if (pvt->fam == 0xf)
1514 return;
1515
1516 if (!dram_rw(pvt, range))
1517 return;
1518
1519 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1520 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1521
1522
1523 if (pvt->fam != 0x15)
1524 return;
1525
1526 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1527 if (WARN_ON(!nb))
1528 return;
1529
1530 if (pvt->model == 0x60)
1531 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1532 else if (pvt->model == 0x30)
1533 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1534 else
1535 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1536
1537 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1538 if (WARN_ON(!f1))
1539 return;
1540
1541 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1542
1543 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1544
1545
1546 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1547
1548 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1549
1550
1551 pvt->ranges[range].lim.hi |= llim >> 13;
1552
1553 pci_dev_put(f1);
1554}
1555
1556static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1557 struct err_info *err)
1558{
1559 struct amd64_pvt *pvt = mci->pvt_info;
1560
1561 error_address_to_page_and_offset(sys_addr, err);
1562
1563
1564
1565
1566
1567 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1568 if (!err->src_mci) {
1569 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1570 (unsigned long)sys_addr);
1571 err->err_code = ERR_NODE;
1572 return;
1573 }
1574
1575
1576 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1577 if (err->csrow < 0) {
1578 err->err_code = ERR_CSROW;
1579 return;
1580 }
1581
1582
1583 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1584 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1585 if (err->channel < 0) {
1586
1587
1588
1589
1590
1591 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1592 "possible error reporting race\n",
1593 err->syndrome);
1594 err->err_code = ERR_CHANNEL;
1595 return;
1596 }
1597 } else {
1598
1599
1600
1601
1602
1603
1604
1605
1606 err->channel = ((sys_addr & BIT(3)) != 0);
1607 }
1608}
1609
1610static int ddr2_cs_size(unsigned i, bool dct_width)
1611{
1612 unsigned shift = 0;
1613
1614 if (i <= 2)
1615 shift = i;
1616 else if (!(i & 0x1))
1617 shift = i >> 1;
1618 else
1619 shift = (i + 1) >> 1;
1620
1621 return 128 << (shift + !!dct_width);
1622}
1623
1624static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1625 unsigned cs_mode, int cs_mask_nr)
1626{
1627 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1628
1629 if (pvt->ext_model >= K8_REV_F) {
1630 WARN_ON(cs_mode > 11);
1631 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1632 }
1633 else if (pvt->ext_model >= K8_REV_D) {
1634 unsigned diff;
1635 WARN_ON(cs_mode > 10);
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1662
1663 return 32 << (cs_mode - diff);
1664 }
1665 else {
1666 WARN_ON(cs_mode > 6);
1667 return 32 << cs_mode;
1668 }
1669}
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679static int f1x_early_channel_count(struct amd64_pvt *pvt)
1680{
1681 int i, j, channels = 0;
1682
1683
1684 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1685 return 2;
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1696
1697
1698
1699
1700
1701
1702 for (i = 0; i < 2; i++) {
1703 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1704
1705 for (j = 0; j < 4; j++) {
1706 if (DBAM_DIMM(j, dbam) > 0) {
1707 channels++;
1708 break;
1709 }
1710 }
1711 }
1712
1713 if (channels > 2)
1714 channels = 2;
1715
1716 amd64_info("MCT channel count: %d\n", channels);
1717
1718 return channels;
1719}
1720
1721static int f17_early_channel_count(struct amd64_pvt *pvt)
1722{
1723 int i, channels = 0;
1724
1725
1726 for_each_umc(i)
1727 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1728
1729 amd64_info("MCT channel count: %d\n", channels);
1730
1731 return channels;
1732}
1733
1734static int ddr3_cs_size(unsigned i, bool dct_width)
1735{
1736 unsigned shift = 0;
1737 int cs_size = 0;
1738
1739 if (i == 0 || i == 3 || i == 4)
1740 cs_size = -1;
1741 else if (i <= 2)
1742 shift = i;
1743 else if (i == 12)
1744 shift = 7;
1745 else if (!(i & 0x1))
1746 shift = i >> 1;
1747 else
1748 shift = (i + 1) >> 1;
1749
1750 if (cs_size != -1)
1751 cs_size = (128 * (1 << !!dct_width)) << shift;
1752
1753 return cs_size;
1754}
1755
1756static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1757{
1758 unsigned shift = 0;
1759 int cs_size = 0;
1760
1761 if (i < 4 || i == 6)
1762 cs_size = -1;
1763 else if (i == 12)
1764 shift = 7;
1765 else if (!(i & 0x1))
1766 shift = i >> 1;
1767 else
1768 shift = (i + 1) >> 1;
1769
1770 if (cs_size != -1)
1771 cs_size = rank_multiply * (128 << shift);
1772
1773 return cs_size;
1774}
1775
1776static int ddr4_cs_size(unsigned i)
1777{
1778 int cs_size = 0;
1779
1780 if (i == 0)
1781 cs_size = -1;
1782 else if (i == 1)
1783 cs_size = 1024;
1784 else
1785
1786 cs_size = 1024 * (1 << (i >> 1));
1787
1788 return cs_size;
1789}
1790
1791static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1792 unsigned cs_mode, int cs_mask_nr)
1793{
1794 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1795
1796 WARN_ON(cs_mode > 11);
1797
1798 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1799 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1800 else
1801 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1802}
1803
1804
1805
1806
1807static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1808 unsigned cs_mode, int cs_mask_nr)
1809{
1810 WARN_ON(cs_mode > 12);
1811
1812 return ddr3_cs_size(cs_mode, false);
1813}
1814
1815
1816static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1817 unsigned cs_mode, int cs_mask_nr)
1818{
1819 int cs_size;
1820 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1821
1822 WARN_ON(cs_mode > 12);
1823
1824 if (pvt->dram_type == MEM_DDR4) {
1825 if (cs_mode > 9)
1826 return -1;
1827
1828 cs_size = ddr4_cs_size(cs_mode);
1829 } else if (pvt->dram_type == MEM_LRDDR3) {
1830 unsigned rank_multiply = dcsm & 0xf;
1831
1832 if (rank_multiply == 3)
1833 rank_multiply = 4;
1834 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1835 } else {
1836
1837 if (cs_mode == 0x1)
1838 return -1;
1839
1840 cs_size = ddr3_cs_size(cs_mode, false);
1841 }
1842
1843 return cs_size;
1844}
1845
1846
1847
1848
1849static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1850 unsigned cs_mode, int cs_mask_nr)
1851{
1852 WARN_ON(cs_mode > 12);
1853
1854 if (cs_mode == 6 || cs_mode == 8 ||
1855 cs_mode == 9 || cs_mode == 12)
1856 return -1;
1857 else
1858 return ddr3_cs_size(cs_mode, false);
1859}
1860
1861static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1862 unsigned int cs_mode, int csrow_nr)
1863{
1864 u32 addr_mask_orig, addr_mask_deinterleaved;
1865 u32 msb, weight, num_zero_bits;
1866 int dimm, size = 0;
1867
1868
1869 if (!cs_mode)
1870 return size;
1871
1872
1873 if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
1874 return size;
1875
1876
1877 if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
1878 return size;
1879
1880
1881
1882
1883
1884
1885 dimm = csrow_nr >> 1;
1886
1887
1888 if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
1889 addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
1890 else
1891 addr_mask_orig = pvt->csels[umc].csmasks[dimm];
1892
1893
1894
1895
1896
1897
1898
1899
1900 msb = fls(addr_mask_orig) - 1;
1901 weight = hweight_long(addr_mask_orig);
1902 num_zero_bits = msb - weight;
1903
1904
1905 addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
1906
1907 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
1908 edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
1909 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
1910
1911
1912 size = (addr_mask_deinterleaved >> 2) + 1;
1913
1914
1915 return size >> 10;
1916}
1917
1918static void read_dram_ctl_register(struct amd64_pvt *pvt)
1919{
1920
1921 if (pvt->fam == 0xf)
1922 return;
1923
1924 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1925 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1926 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1927
1928 edac_dbg(0, " DCTs operate in %s mode\n",
1929 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1930
1931 if (!dct_ganging_enabled(pvt))
1932 edac_dbg(0, " Address range split per DCT: %s\n",
1933 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1934
1935 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1936 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1937 (dct_memory_cleared(pvt) ? "yes" : "no"));
1938
1939 edac_dbg(0, " channel interleave: %s, "
1940 "interleave bits selector: 0x%x\n",
1941 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1942 dct_sel_interleave_addr(pvt));
1943 }
1944
1945 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1946}
1947
1948
1949
1950
1951
1952static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1953 u8 intlv_en, int num_dcts_intlv,
1954 u32 dct_sel)
1955{
1956 u8 channel = 0;
1957 u8 select;
1958
1959 if (!(intlv_en))
1960 return (u8)(dct_sel);
1961
1962 if (num_dcts_intlv == 2) {
1963 select = (sys_addr >> 8) & 0x3;
1964 channel = select ? 0x3 : 0;
1965 } else if (num_dcts_intlv == 4) {
1966 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1967 switch (intlv_addr) {
1968 case 0x4:
1969 channel = (sys_addr >> 8) & 0x3;
1970 break;
1971 case 0x5:
1972 channel = (sys_addr >> 9) & 0x3;
1973 break;
1974 }
1975 }
1976 return channel;
1977}
1978
1979
1980
1981
1982
1983static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1984 bool hi_range_sel, u8 intlv_en)
1985{
1986 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1987
1988 if (dct_ganging_enabled(pvt))
1989 return 0;
1990
1991 if (hi_range_sel)
1992 return dct_sel_high;
1993
1994
1995
1996
1997 if (dct_interleave_enabled(pvt)) {
1998 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1999
2000
2001 if (!intlv_addr)
2002 return sys_addr >> 6 & 1;
2003
2004 if (intlv_addr & 0x2) {
2005 u8 shift = intlv_addr & 0x1 ? 9 : 6;
2006 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
2007
2008 return ((sys_addr >> shift) & 1) ^ temp;
2009 }
2010
2011 if (intlv_addr & 0x4) {
2012 u8 shift = intlv_addr & 0x1 ? 9 : 8;
2013
2014 return (sys_addr >> shift) & 1;
2015 }
2016
2017 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2018 }
2019
2020 if (dct_high_range_enabled(pvt))
2021 return ~dct_sel_high & 1;
2022
2023 return 0;
2024}
2025
2026
2027static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
2028 u64 sys_addr, bool hi_rng,
2029 u32 dct_sel_base_addr)
2030{
2031 u64 chan_off;
2032 u64 dram_base = get_dram_base(pvt, range);
2033 u64 hole_off = f10_dhar_offset(pvt);
2034 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
2035
2036 if (hi_rng) {
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048 if ((!(dct_sel_base_addr >> 16) ||
2049 dct_sel_base_addr < dhar_base(pvt)) &&
2050 dhar_valid(pvt) &&
2051 (sys_addr >= BIT_64(32)))
2052 chan_off = hole_off;
2053 else
2054 chan_off = dct_sel_base_off;
2055 } else {
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
2066 chan_off = hole_off;
2067 else
2068 chan_off = dram_base;
2069 }
2070
2071 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
2072}
2073
2074
2075
2076
2077
2078static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
2079{
2080 int tmp_cs;
2081
2082 if (online_spare_swap_done(pvt, dct) &&
2083 csrow == online_spare_bad_dramcs(pvt, dct)) {
2084
2085 for_each_chip_select(tmp_cs, dct, pvt) {
2086 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2087 csrow = tmp_cs;
2088 break;
2089 }
2090 }
2091 }
2092 return csrow;
2093}
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
2104{
2105 struct mem_ctl_info *mci;
2106 struct amd64_pvt *pvt;
2107 u64 cs_base, cs_mask;
2108 int cs_found = -EINVAL;
2109 int csrow;
2110
2111 mci = edac_mc_find(nid);
2112 if (!mci)
2113 return cs_found;
2114
2115 pvt = mci->pvt_info;
2116
2117 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
2118
2119 for_each_chip_select(csrow, dct, pvt) {
2120 if (!csrow_enabled(csrow, dct, pvt))
2121 continue;
2122
2123 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
2124
2125 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2126 csrow, cs_base, cs_mask);
2127
2128 cs_mask = ~cs_mask;
2129
2130 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2131 (in_addr & cs_mask), (cs_base & cs_mask));
2132
2133 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
2134 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
2135 cs_found = csrow;
2136 break;
2137 }
2138 cs_found = f10_process_possible_spare(pvt, dct, csrow);
2139
2140 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
2141 break;
2142 }
2143 }
2144 return cs_found;
2145}
2146
2147
2148
2149
2150
2151
2152static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
2153{
2154 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
2155
2156 if (pvt->fam == 0x10) {
2157
2158 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
2159 return sys_addr;
2160 }
2161
2162 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
2163
2164 if (!(swap_reg & 0x1))
2165 return sys_addr;
2166
2167 swap_base = (swap_reg >> 3) & 0x7f;
2168 swap_limit = (swap_reg >> 11) & 0x7f;
2169 rgn_size = (swap_reg >> 20) & 0x7f;
2170 tmp_addr = sys_addr >> 27;
2171
2172 if (!(sys_addr >> 34) &&
2173 (((tmp_addr >= swap_base) &&
2174 (tmp_addr <= swap_limit)) ||
2175 (tmp_addr < rgn_size)))
2176 return sys_addr ^ (u64)swap_base << 27;
2177
2178 return sys_addr;
2179}
2180
2181
2182static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2183 u64 sys_addr, int *chan_sel)
2184{
2185 int cs_found = -EINVAL;
2186 u64 chan_addr;
2187 u32 dct_sel_base;
2188 u8 channel;
2189 bool high_range = false;
2190
2191 u8 node_id = dram_dst_node(pvt, range);
2192 u8 intlv_en = dram_intlv_en(pvt, range);
2193 u32 intlv_sel = dram_intlv_sel(pvt, range);
2194
2195 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2196 range, sys_addr, get_dram_limit(pvt, range));
2197
2198 if (dhar_valid(pvt) &&
2199 dhar_base(pvt) <= sys_addr &&
2200 sys_addr < BIT_64(32)) {
2201 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2202 sys_addr);
2203 return -EINVAL;
2204 }
2205
2206 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
2207 return -EINVAL;
2208
2209 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
2210
2211 dct_sel_base = dct_sel_baseaddr(pvt);
2212
2213
2214
2215
2216
2217 if (dct_high_range_enabled(pvt) &&
2218 !dct_ganging_enabled(pvt) &&
2219 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
2220 high_range = true;
2221
2222 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
2223
2224 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
2225 high_range, dct_sel_base);
2226
2227
2228 if (intlv_en)
2229 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2230 (chan_addr & 0xfff);
2231
2232
2233 if (dct_interleave_enabled(pvt) &&
2234 !dct_high_range_enabled(pvt) &&
2235 !dct_ganging_enabled(pvt)) {
2236
2237 if (dct_sel_interleave_addr(pvt) != 1) {
2238 if (dct_sel_interleave_addr(pvt) == 0x3)
2239
2240 chan_addr = ((chan_addr >> 10) << 9) |
2241 (chan_addr & 0x1ff);
2242 else
2243
2244 chan_addr = ((chan_addr >> 7) << 6) |
2245 (chan_addr & 0x3f);
2246 } else
2247
2248 chan_addr = ((chan_addr >> 13) << 12) |
2249 (chan_addr & 0xfff);
2250 }
2251
2252 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2253
2254 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
2255
2256 if (cs_found >= 0)
2257 *chan_sel = channel;
2258
2259 return cs_found;
2260}
2261
2262static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2263 u64 sys_addr, int *chan_sel)
2264{
2265 int cs_found = -EINVAL;
2266 int num_dcts_intlv = 0;
2267 u64 chan_addr, chan_offset;
2268 u64 dct_base, dct_limit;
2269 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
2270 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
2271
2272 u64 dhar_offset = f10_dhar_offset(pvt);
2273 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2274 u8 node_id = dram_dst_node(pvt, range);
2275 u8 intlv_en = dram_intlv_en(pvt, range);
2276
2277 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
2278 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
2279
2280 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
2281 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
2282
2283 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2284 range, sys_addr, get_dram_limit(pvt, range));
2285
2286 if (!(get_dram_base(pvt, range) <= sys_addr) &&
2287 !(get_dram_limit(pvt, range) >= sys_addr))
2288 return -EINVAL;
2289
2290 if (dhar_valid(pvt) &&
2291 dhar_base(pvt) <= sys_addr &&
2292 sys_addr < BIT_64(32)) {
2293 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2294 sys_addr);
2295 return -EINVAL;
2296 }
2297
2298
2299 dct_base = (u64) dct_sel_baseaddr(pvt);
2300 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
2301
2302 if (!(dct_cont_base_reg & BIT(0)) &&
2303 !(dct_base <= (sys_addr >> 27) &&
2304 dct_limit >= (sys_addr >> 27)))
2305 return -EINVAL;
2306
2307
2308 num_dcts_intlv = (int) hweight8(intlv_en);
2309
2310 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
2311 return -EINVAL;
2312
2313 if (pvt->model >= 0x60)
2314 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2315 else
2316 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
2317 num_dcts_intlv, dct_sel);
2318
2319
2320 if (channel > 3)
2321 return -EINVAL;
2322
2323 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
2324
2325
2326 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
2327 chan_offset = dhar_offset;
2328 else
2329 chan_offset = dct_base << 27;
2330
2331 chan_addr = sys_addr - chan_offset;
2332
2333
2334 if (num_dcts_intlv == 2) {
2335 if (intlv_addr == 0x4)
2336 chan_addr = ((chan_addr >> 9) << 8) |
2337 (chan_addr & 0xff);
2338 else if (intlv_addr == 0x5)
2339 chan_addr = ((chan_addr >> 10) << 9) |
2340 (chan_addr & 0x1ff);
2341 else
2342 return -EINVAL;
2343
2344 } else if (num_dcts_intlv == 4) {
2345 if (intlv_addr == 0x4)
2346 chan_addr = ((chan_addr >> 10) << 8) |
2347 (chan_addr & 0xff);
2348 else if (intlv_addr == 0x5)
2349 chan_addr = ((chan_addr >> 11) << 9) |
2350 (chan_addr & 0x1ff);
2351 else
2352 return -EINVAL;
2353 }
2354
2355 if (dct_offset_en) {
2356 amd64_read_pci_cfg(pvt->F1,
2357 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2358 &tmp);
2359 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
2360 }
2361
2362 f15h_select_dct(pvt, channel);
2363
2364 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374 alias_channel = (channel == 3) ? 1 : channel;
2375
2376 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2377
2378 if (cs_found >= 0)
2379 *chan_sel = alias_channel;
2380
2381 return cs_found;
2382}
2383
2384static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2385 u64 sys_addr,
2386 int *chan_sel)
2387{
2388 int cs_found = -EINVAL;
2389 unsigned range;
2390
2391 for (range = 0; range < DRAM_RANGES; range++) {
2392 if (!dram_rw(pvt, range))
2393 continue;
2394
2395 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2396 cs_found = f15_m30h_match_to_this_node(pvt, range,
2397 sys_addr,
2398 chan_sel);
2399
2400 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2401 (get_dram_limit(pvt, range) >= sys_addr)) {
2402 cs_found = f1x_match_to_this_node(pvt, range,
2403 sys_addr, chan_sel);
2404 if (cs_found >= 0)
2405 break;
2406 }
2407 }
2408 return cs_found;
2409}
2410
2411
2412
2413
2414
2415
2416
2417
2418static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2419 struct err_info *err)
2420{
2421 struct amd64_pvt *pvt = mci->pvt_info;
2422
2423 error_address_to_page_and_offset(sys_addr, err);
2424
2425 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2426 if (err->csrow < 0) {
2427 err->err_code = ERR_CSROW;
2428 return;
2429 }
2430
2431
2432
2433
2434
2435
2436 if (dct_ganging_enabled(pvt))
2437 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2438}
2439
2440
2441
2442
2443
2444static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2445{
2446 int dimm, size0, size1;
2447 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2448 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2449
2450 if (pvt->fam == 0xf) {
2451
2452 if (pvt->ext_model < K8_REV_F)
2453 return;
2454 else
2455 WARN_ON(ctrl != 0);
2456 }
2457
2458 if (pvt->fam == 0x10) {
2459 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2460 : pvt->dbam0;
2461 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2462 pvt->csels[1].csbases :
2463 pvt->csels[0].csbases;
2464 } else if (ctrl) {
2465 dbam = pvt->dbam0;
2466 dcsb = pvt->csels[1].csbases;
2467 }
2468 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2469 ctrl, dbam);
2470
2471 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2472
2473
2474 for (dimm = 0; dimm < 4; dimm++) {
2475
2476 size0 = 0;
2477 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2478
2479
2480
2481
2482
2483
2484 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2485 DBAM_DIMM(dimm, dbam),
2486 dimm);
2487
2488 size1 = 0;
2489 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2490 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2491 DBAM_DIMM(dimm, dbam),
2492 dimm);
2493
2494 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2495 dimm * 2, size0,
2496 dimm * 2 + 1, size1);
2497 }
2498}
2499
2500static struct amd64_family_type family_types[] = {
2501 [K8_CPUS] = {
2502 .ctl_name = "K8",
2503 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2504 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2505 .max_mcs = 2,
2506 .ops = {
2507 .early_channel_count = k8_early_channel_count,
2508 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2509 .dbam_to_cs = k8_dbam_to_chip_select,
2510 }
2511 },
2512 [F10_CPUS] = {
2513 .ctl_name = "F10h",
2514 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2515 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2516 .max_mcs = 2,
2517 .ops = {
2518 .early_channel_count = f1x_early_channel_count,
2519 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2520 .dbam_to_cs = f10_dbam_to_chip_select,
2521 }
2522 },
2523 [F15_CPUS] = {
2524 .ctl_name = "F15h",
2525 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2526 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2527 .max_mcs = 2,
2528 .ops = {
2529 .early_channel_count = f1x_early_channel_count,
2530 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2531 .dbam_to_cs = f15_dbam_to_chip_select,
2532 }
2533 },
2534 [F15_M30H_CPUS] = {
2535 .ctl_name = "F15h_M30h",
2536 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2537 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2538 .max_mcs = 2,
2539 .ops = {
2540 .early_channel_count = f1x_early_channel_count,
2541 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2542 .dbam_to_cs = f16_dbam_to_chip_select,
2543 }
2544 },
2545 [F15_M60H_CPUS] = {
2546 .ctl_name = "F15h_M60h",
2547 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2548 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2549 .max_mcs = 2,
2550 .ops = {
2551 .early_channel_count = f1x_early_channel_count,
2552 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2553 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2554 }
2555 },
2556 [F16_CPUS] = {
2557 .ctl_name = "F16h",
2558 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2559 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2560 .max_mcs = 2,
2561 .ops = {
2562 .early_channel_count = f1x_early_channel_count,
2563 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2564 .dbam_to_cs = f16_dbam_to_chip_select,
2565 }
2566 },
2567 [F16_M30H_CPUS] = {
2568 .ctl_name = "F16h_M30h",
2569 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2570 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2571 .max_mcs = 2,
2572 .ops = {
2573 .early_channel_count = f1x_early_channel_count,
2574 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2575 .dbam_to_cs = f16_dbam_to_chip_select,
2576 }
2577 },
2578 [F17_CPUS] = {
2579 .ctl_name = "F17h",
2580 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2581 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2582 .max_mcs = 2,
2583 .ops = {
2584 .early_channel_count = f17_early_channel_count,
2585 .dbam_to_cs = f17_addr_mask_to_cs_size,
2586 }
2587 },
2588 [F17_M10H_CPUS] = {
2589 .ctl_name = "F17h_M10h",
2590 .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2591 .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
2592 .max_mcs = 2,
2593 .ops = {
2594 .early_channel_count = f17_early_channel_count,
2595 .dbam_to_cs = f17_addr_mask_to_cs_size,
2596 }
2597 },
2598 [F17_M30H_CPUS] = {
2599 .ctl_name = "F17h_M30h",
2600 .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2601 .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
2602 .max_mcs = 8,
2603 .ops = {
2604 .early_channel_count = f17_early_channel_count,
2605 .dbam_to_cs = f17_addr_mask_to_cs_size,
2606 }
2607 },
2608 [F17_M60H_CPUS] = {
2609 .ctl_name = "F17h_M60h",
2610 .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0,
2611 .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6,
2612 .max_mcs = 2,
2613 .ops = {
2614 .early_channel_count = f17_early_channel_count,
2615 .dbam_to_cs = f17_addr_mask_to_cs_size,
2616 }
2617 },
2618 [F17_M70H_CPUS] = {
2619 .ctl_name = "F17h_M70h",
2620 .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
2621 .f6_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F6,
2622 .max_mcs = 2,
2623 .ops = {
2624 .early_channel_count = f17_early_channel_count,
2625 .dbam_to_cs = f17_addr_mask_to_cs_size,
2626 }
2627 },
2628 [F19_CPUS] = {
2629 .ctl_name = "F19h",
2630 .f0_id = PCI_DEVICE_ID_AMD_19H_DF_F0,
2631 .f6_id = PCI_DEVICE_ID_AMD_19H_DF_F6,
2632 .max_mcs = 8,
2633 .ops = {
2634 .early_channel_count = f17_early_channel_count,
2635 .dbam_to_cs = f17_addr_mask_to_cs_size,
2636 }
2637 },
2638};
2639
2640
2641
2642
2643
2644
2645
2646
2647static const u16 x4_vectors[] = {
2648 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2649 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2650 0x0001, 0x0002, 0x0004, 0x0008,
2651 0x1013, 0x3032, 0x4044, 0x8088,
2652 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2653 0x4857, 0xc4fe, 0x13cc, 0x3288,
2654 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2655 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2656 0x15c1, 0x2a42, 0x89ac, 0x4758,
2657 0x2b03, 0x1602, 0x4f0c, 0xca08,
2658 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2659 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2660 0x2b87, 0x164e, 0x642c, 0xdc18,
2661 0x40b9, 0x80de, 0x1094, 0x20e8,
2662 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2663 0x11c1, 0x2242, 0x84ac, 0x4c58,
2664 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2665 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2666 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2667 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2668 0x16b3, 0x3d62, 0x4f34, 0x8518,
2669 0x1e2f, 0x391a, 0x5cac, 0xf858,
2670 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2671 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2672 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2673 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2674 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2675 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2676 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2677 0x185d, 0x2ca6, 0x7914, 0x9e28,
2678 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2679 0x4199, 0x82ee, 0x19f4, 0x2e58,
2680 0x4807, 0xc40e, 0x130c, 0x3208,
2681 0x1905, 0x2e0a, 0x5804, 0xac08,
2682 0x213f, 0x132a, 0xadfc, 0x5ba8,
2683 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2684};
2685
2686static const u16 x8_vectors[] = {
2687 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2688 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2689 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2690 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2691 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2692 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2693 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2694 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2695 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2696 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2697 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2698 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2699 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2700 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2701 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2702 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2703 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2704 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2705 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2706};
2707
2708static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2709 unsigned v_dim)
2710{
2711 unsigned int i, err_sym;
2712
2713 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2714 u16 s = syndrome;
2715 unsigned v_idx = err_sym * v_dim;
2716 unsigned v_end = (err_sym + 1) * v_dim;
2717
2718
2719 for (i = 1; i < (1U << 16); i <<= 1) {
2720
2721
2722 if (v_idx < v_end && vectors[v_idx] & i) {
2723 u16 ev_comp = vectors[v_idx++];
2724
2725
2726 if (s & i) {
2727
2728 s ^= ev_comp;
2729
2730 if (!s)
2731 return err_sym;
2732 }
2733
2734 } else if (s & i)
2735
2736 break;
2737 }
2738 }
2739
2740 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2741 return -1;
2742}
2743
2744static int map_err_sym_to_channel(int err_sym, int sym_size)
2745{
2746 if (sym_size == 4)
2747 switch (err_sym) {
2748 case 0x20:
2749 case 0x21:
2750 return 0;
2751 case 0x22:
2752 case 0x23:
2753 return 1;
2754 default:
2755 return err_sym >> 4;
2756 }
2757
2758 else
2759 switch (err_sym) {
2760
2761 case 0x10:
2762 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2763 err_sym);
2764 return -1;
2765 case 0x11:
2766 return 0;
2767 case 0x12:
2768 return 1;
2769 default:
2770 return err_sym >> 3;
2771 }
2772 return -1;
2773}
2774
2775static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2776{
2777 struct amd64_pvt *pvt = mci->pvt_info;
2778 int err_sym = -1;
2779
2780 if (pvt->ecc_sym_sz == 8)
2781 err_sym = decode_syndrome(syndrome, x8_vectors,
2782 ARRAY_SIZE(x8_vectors),
2783 pvt->ecc_sym_sz);
2784 else if (pvt->ecc_sym_sz == 4)
2785 err_sym = decode_syndrome(syndrome, x4_vectors,
2786 ARRAY_SIZE(x4_vectors),
2787 pvt->ecc_sym_sz);
2788 else {
2789 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2790 return err_sym;
2791 }
2792
2793 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2794}
2795
2796static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2797 u8 ecc_type)
2798{
2799 enum hw_event_mc_err_type err_type;
2800 const char *string;
2801
2802 if (ecc_type == 2)
2803 err_type = HW_EVENT_ERR_CORRECTED;
2804 else if (ecc_type == 1)
2805 err_type = HW_EVENT_ERR_UNCORRECTED;
2806 else if (ecc_type == 3)
2807 err_type = HW_EVENT_ERR_DEFERRED;
2808 else {
2809 WARN(1, "Something is rotten in the state of Denmark.\n");
2810 return;
2811 }
2812
2813 switch (err->err_code) {
2814 case DECODE_OK:
2815 string = "";
2816 break;
2817 case ERR_NODE:
2818 string = "Failed to map error addr to a node";
2819 break;
2820 case ERR_CSROW:
2821 string = "Failed to map error addr to a csrow";
2822 break;
2823 case ERR_CHANNEL:
2824 string = "Unknown syndrome - possible error reporting race";
2825 break;
2826 case ERR_SYND:
2827 string = "MCA_SYND not valid - unknown syndrome and csrow";
2828 break;
2829 case ERR_NORM_ADDR:
2830 string = "Cannot decode normalized address";
2831 break;
2832 default:
2833 string = "WTF error";
2834 break;
2835 }
2836
2837 edac_mc_handle_error(err_type, mci, 1,
2838 err->page, err->offset, err->syndrome,
2839 err->csrow, err->channel, -1,
2840 string, "");
2841}
2842
2843static inline void decode_bus_error(int node_id, struct mce *m)
2844{
2845 struct mem_ctl_info *mci;
2846 struct amd64_pvt *pvt;
2847 u8 ecc_type = (m->status >> 45) & 0x3;
2848 u8 xec = XEC(m->status, 0x1f);
2849 u16 ec = EC(m->status);
2850 u64 sys_addr;
2851 struct err_info err;
2852
2853 mci = edac_mc_find(node_id);
2854 if (!mci)
2855 return;
2856
2857 pvt = mci->pvt_info;
2858
2859
2860 if (PP(ec) == NBSL_PP_OBS)
2861 return;
2862
2863
2864 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2865 return;
2866
2867 memset(&err, 0, sizeof(err));
2868
2869 sys_addr = get_error_address(pvt, m);
2870
2871 if (ecc_type == 2)
2872 err.syndrome = extract_syndrome(m->status);
2873
2874 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2875
2876 __log_ecc_error(mci, &err, ecc_type);
2877}
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888static int find_umc_channel(struct mce *m)
2889{
2890 return (m->ipid & GENMASK(31, 0)) >> 20;
2891}
2892
2893static void decode_umc_error(int node_id, struct mce *m)
2894{
2895 u8 ecc_type = (m->status >> 45) & 0x3;
2896 struct mem_ctl_info *mci;
2897 struct amd64_pvt *pvt;
2898 struct err_info err;
2899 u64 sys_addr;
2900
2901 mci = edac_mc_find(node_id);
2902 if (!mci)
2903 return;
2904
2905 pvt = mci->pvt_info;
2906
2907 memset(&err, 0, sizeof(err));
2908
2909 if (m->status & MCI_STATUS_DEFERRED)
2910 ecc_type = 3;
2911
2912 err.channel = find_umc_channel(m);
2913
2914 if (!(m->status & MCI_STATUS_SYNDV)) {
2915 err.err_code = ERR_SYND;
2916 goto log_error;
2917 }
2918
2919 if (ecc_type == 2) {
2920 u8 length = (m->synd >> 18) & 0x3f;
2921
2922 if (length)
2923 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2924 else
2925 err.err_code = ERR_CHANNEL;
2926 }
2927
2928 err.csrow = m->synd & 0x7;
2929
2930 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2931 err.err_code = ERR_NORM_ADDR;
2932 goto log_error;
2933 }
2934
2935 error_address_to_page_and_offset(sys_addr, &err);
2936
2937log_error:
2938 __log_ecc_error(mci, &err, ecc_type);
2939}
2940
2941
2942
2943
2944
2945
2946static int
2947reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2948{
2949 if (pvt->umc) {
2950 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2951 if (!pvt->F0) {
2952 edac_dbg(1, "F0 not found, device 0x%x\n", pci_id1);
2953 return -ENODEV;
2954 }
2955
2956 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2957 if (!pvt->F6) {
2958 pci_dev_put(pvt->F0);
2959 pvt->F0 = NULL;
2960
2961 edac_dbg(1, "F6 not found: device 0x%x\n", pci_id2);
2962 return -ENODEV;
2963 }
2964
2965 if (!pci_ctl_dev)
2966 pci_ctl_dev = &pvt->F0->dev;
2967
2968 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2969 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2970 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2971
2972 return 0;
2973 }
2974
2975
2976 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2977 if (!pvt->F1) {
2978 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
2979 return -ENODEV;
2980 }
2981
2982
2983 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2984 if (!pvt->F2) {
2985 pci_dev_put(pvt->F1);
2986 pvt->F1 = NULL;
2987
2988 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
2989 return -ENODEV;
2990 }
2991
2992 if (!pci_ctl_dev)
2993 pci_ctl_dev = &pvt->F2->dev;
2994
2995 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2996 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2997 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2998
2999 return 0;
3000}
3001
3002static void free_mc_sibling_devs(struct amd64_pvt *pvt)
3003{
3004 if (pvt->umc) {
3005 pci_dev_put(pvt->F0);
3006 pci_dev_put(pvt->F6);
3007 } else {
3008 pci_dev_put(pvt->F1);
3009 pci_dev_put(pvt->F2);
3010 }
3011}
3012
3013static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3014{
3015 pvt->ecc_sym_sz = 4;
3016
3017 if (pvt->umc) {
3018 u8 i;
3019
3020 for_each_umc(i) {
3021
3022 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3023 if (pvt->umc[i].ecc_ctrl & BIT(9)) {
3024 pvt->ecc_sym_sz = 16;
3025 return;
3026 } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
3027 pvt->ecc_sym_sz = 8;
3028 return;
3029 }
3030 }
3031 }
3032 } else if (pvt->fam >= 0x10) {
3033 u32 tmp;
3034
3035 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3036
3037 if (pvt->fam != 0x16)
3038 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3039
3040
3041 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3042 pvt->ecc_sym_sz = 8;
3043 }
3044}
3045
3046
3047
3048
3049static void __read_mc_regs_df(struct amd64_pvt *pvt)
3050{
3051 u8 nid = pvt->mc_node_id;
3052 struct amd64_umc *umc;
3053 u32 i, umc_base;
3054
3055
3056 for_each_umc(i) {
3057
3058 umc_base = get_umc_base(i);
3059 umc = &pvt->umc[i];
3060
3061 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
3062 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3063 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3064 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
3065 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
3066 }
3067}
3068
3069
3070
3071
3072
3073static void read_mc_regs(struct amd64_pvt *pvt)
3074{
3075 unsigned int range;
3076 u64 msr_val;
3077
3078
3079
3080
3081
3082 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
3083 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
3084
3085
3086 rdmsrl(MSR_AMD64_SYSCFG, msr_val);
3087 if (msr_val & BIT(21)) {
3088 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
3089 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
3090 } else {
3091 edac_dbg(0, " TOP_MEM2 disabled\n");
3092 }
3093
3094 if (pvt->umc) {
3095 __read_mc_regs_df(pvt);
3096 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
3097
3098 goto skip;
3099 }
3100
3101 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
3102
3103 read_dram_ctl_register(pvt);
3104
3105 for (range = 0; range < DRAM_RANGES; range++) {
3106 u8 rw;
3107
3108
3109 read_dram_base_limit_regs(pvt, range);
3110
3111 rw = dram_rw(pvt, range);
3112 if (!rw)
3113 continue;
3114
3115 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
3116 range,
3117 get_dram_base(pvt, range),
3118 get_dram_limit(pvt, range));
3119
3120 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
3121 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
3122 (rw & 0x1) ? "R" : "-",
3123 (rw & 0x2) ? "W" : "-",
3124 dram_intlv_sel(pvt, range),
3125 dram_dst_node(pvt, range));
3126 }
3127
3128 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
3129 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
3130
3131 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
3132
3133 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
3134 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
3135
3136 if (!dct_ganging_enabled(pvt)) {
3137 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
3138 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
3139 }
3140
3141skip:
3142 read_dct_base_mask(pvt);
3143
3144 determine_memory_type(pvt);
3145 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
3146
3147 determine_ecc_sym_sz(pvt);
3148}
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
3185{
3186 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
3187 int csrow_nr = csrow_nr_orig;
3188 u32 cs_mode, nr_pages;
3189
3190 if (!pvt->umc) {
3191 csrow_nr >>= 1;
3192 cs_mode = DBAM_DIMM(csrow_nr, dbam);
3193 } else {
3194 cs_mode = f17_get_cs_mode(csrow_nr >> 1, dct, pvt);
3195 }
3196
3197 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3198 nr_pages <<= 20 - PAGE_SHIFT;
3199
3200 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3201 csrow_nr_orig, dct, cs_mode);
3202 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3203
3204 return nr_pages;
3205}
3206
3207static int init_csrows_df(struct mem_ctl_info *mci)
3208{
3209 struct amd64_pvt *pvt = mci->pvt_info;
3210 enum edac_type edac_mode = EDAC_NONE;
3211 enum dev_type dev_type = DEV_UNKNOWN;
3212 struct dimm_info *dimm;
3213 int empty = 1;
3214 u8 umc, cs;
3215
3216 if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3217 edac_mode = EDAC_S16ECD16ED;
3218 dev_type = DEV_X16;
3219 } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3220 edac_mode = EDAC_S8ECD8ED;
3221 dev_type = DEV_X8;
3222 } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3223 edac_mode = EDAC_S4ECD4ED;
3224 dev_type = DEV_X4;
3225 } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3226 edac_mode = EDAC_SECDED;
3227 }
3228
3229 for_each_umc(umc) {
3230 for_each_chip_select(cs, umc, pvt) {
3231 if (!csrow_enabled(cs, umc, pvt))
3232 continue;
3233
3234 empty = 0;
3235 dimm = mci->csrows[cs]->channels[umc]->dimm;
3236
3237 edac_dbg(1, "MC node: %d, csrow: %d\n",
3238 pvt->mc_node_id, cs);
3239
3240 dimm->nr_pages = get_csrow_nr_pages(pvt, umc, cs);
3241 dimm->mtype = pvt->dram_type;
3242 dimm->edac_mode = edac_mode;
3243 dimm->dtype = dev_type;
3244 dimm->grain = 64;
3245 }
3246 }
3247
3248 return empty;
3249}
3250
3251
3252
3253
3254
3255static int init_csrows(struct mem_ctl_info *mci)
3256{
3257 struct amd64_pvt *pvt = mci->pvt_info;
3258 enum edac_type edac_mode = EDAC_NONE;
3259 struct csrow_info *csrow;
3260 struct dimm_info *dimm;
3261 int i, j, empty = 1;
3262 int nr_pages = 0;
3263 u32 val;
3264
3265 if (pvt->umc)
3266 return init_csrows_df(mci);
3267
3268 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
3269
3270 pvt->nbcfg = val;
3271
3272 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3273 pvt->mc_node_id, val,
3274 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
3275
3276
3277
3278
3279 for_each_chip_select(i, 0, pvt) {
3280 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
3281 bool row_dct1 = false;
3282
3283 if (pvt->fam != 0xf)
3284 row_dct1 = !!csrow_enabled(i, 1, pvt);
3285
3286 if (!row_dct0 && !row_dct1)
3287 continue;
3288
3289 csrow = mci->csrows[i];
3290 empty = 0;
3291
3292 edac_dbg(1, "MC node: %d, csrow: %d\n",
3293 pvt->mc_node_id, i);
3294
3295 if (row_dct0) {
3296 nr_pages = get_csrow_nr_pages(pvt, 0, i);
3297 csrow->channels[0]->dimm->nr_pages = nr_pages;
3298 }
3299
3300
3301 if (pvt->fam != 0xf && row_dct1) {
3302 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
3303
3304 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
3305 nr_pages += row_dct1_pages;
3306 }
3307
3308 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
3309
3310
3311 if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
3312 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
3313 ? EDAC_S4ECD4ED
3314 : EDAC_SECDED;
3315 }
3316
3317 for (j = 0; j < pvt->channel_count; j++) {
3318 dimm = csrow->channels[j]->dimm;
3319 dimm->mtype = pvt->dram_type;
3320 dimm->edac_mode = edac_mode;
3321 dimm->grain = 64;
3322 }
3323 }
3324
3325 return empty;
3326}
3327
3328
3329static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
3330{
3331 int cpu;
3332
3333 for_each_online_cpu(cpu)
3334 if (topology_die_id(cpu) == nid)
3335 cpumask_set_cpu(cpu, mask);
3336}
3337
3338
3339static bool nb_mce_bank_enabled_on_node(u16 nid)
3340{
3341 cpumask_var_t mask;
3342 int cpu, nbe;
3343 bool ret = false;
3344
3345 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3346 amd64_warn("%s: Error allocating mask\n", __func__);
3347 return false;
3348 }
3349
3350 get_cpus_on_this_dct_cpumask(mask, nid);
3351
3352 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
3353
3354 for_each_cpu(cpu, mask) {
3355 struct msr *reg = per_cpu_ptr(msrs, cpu);
3356 nbe = reg->l & MSR_MCGCTL_NBE;
3357
3358 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3359 cpu, reg->q,
3360 (nbe ? "enabled" : "disabled"));
3361
3362 if (!nbe)
3363 goto out;
3364 }
3365 ret = true;
3366
3367out:
3368 free_cpumask_var(mask);
3369 return ret;
3370}
3371
3372static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
3373{
3374 cpumask_var_t cmask;
3375 int cpu;
3376
3377 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
3378 amd64_warn("%s: error allocating mask\n", __func__);
3379 return -ENOMEM;
3380 }
3381
3382 get_cpus_on_this_dct_cpumask(cmask, nid);
3383
3384 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3385
3386 for_each_cpu(cpu, cmask) {
3387
3388 struct msr *reg = per_cpu_ptr(msrs, cpu);
3389
3390 if (on) {
3391 if (reg->l & MSR_MCGCTL_NBE)
3392 s->flags.nb_mce_enable = 1;
3393
3394 reg->l |= MSR_MCGCTL_NBE;
3395 } else {
3396
3397
3398
3399 if (!s->flags.nb_mce_enable)
3400 reg->l &= ~MSR_MCGCTL_NBE;
3401 }
3402 }
3403 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3404
3405 free_cpumask_var(cmask);
3406
3407 return 0;
3408}
3409
3410static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3411 struct pci_dev *F3)
3412{
3413 bool ret = true;
3414 u32 value, mask = 0x3;
3415
3416 if (toggle_ecc_err_reporting(s, nid, ON)) {
3417 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3418 return false;
3419 }
3420
3421 amd64_read_pci_cfg(F3, NBCTL, &value);
3422
3423 s->old_nbctl = value & mask;
3424 s->nbctl_valid = true;
3425
3426 value |= mask;
3427 amd64_write_pci_cfg(F3, NBCTL, value);
3428
3429 amd64_read_pci_cfg(F3, NBCFG, &value);
3430
3431 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3432 nid, value, !!(value & NBCFG_ECC_ENABLE));
3433
3434 if (!(value & NBCFG_ECC_ENABLE)) {
3435 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3436
3437 s->flags.nb_ecc_prev = 0;
3438
3439
3440 value |= NBCFG_ECC_ENABLE;
3441 amd64_write_pci_cfg(F3, NBCFG, value);
3442
3443 amd64_read_pci_cfg(F3, NBCFG, &value);
3444
3445 if (!(value & NBCFG_ECC_ENABLE)) {
3446 amd64_warn("Hardware rejected DRAM ECC enable,"
3447 "check memory DIMM configuration.\n");
3448 ret = false;
3449 } else {
3450 amd64_info("Hardware accepted DRAM ECC Enable\n");
3451 }
3452 } else {
3453 s->flags.nb_ecc_prev = 1;
3454 }
3455
3456 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3457 nid, value, !!(value & NBCFG_ECC_ENABLE));
3458
3459 return ret;
3460}
3461
3462static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3463 struct pci_dev *F3)
3464{
3465 u32 value, mask = 0x3;
3466
3467 if (!s->nbctl_valid)
3468 return;
3469
3470 amd64_read_pci_cfg(F3, NBCTL, &value);
3471 value &= ~mask;
3472 value |= s->old_nbctl;
3473
3474 amd64_write_pci_cfg(F3, NBCTL, value);
3475
3476
3477 if (!s->flags.nb_ecc_prev) {
3478 amd64_read_pci_cfg(F3, NBCFG, &value);
3479 value &= ~NBCFG_ECC_ENABLE;
3480 amd64_write_pci_cfg(F3, NBCFG, value);
3481 }
3482
3483
3484 if (toggle_ecc_err_reporting(s, nid, OFF))
3485 amd64_warn("Error restoring NB MCGCTL settings!\n");
3486}
3487
3488static bool ecc_enabled(struct amd64_pvt *pvt)
3489{
3490 u16 nid = pvt->mc_node_id;
3491 bool nb_mce_en = false;
3492 u8 ecc_en = 0, i;
3493 u32 value;
3494
3495 if (boot_cpu_data.x86 >= 0x17) {
3496 u8 umc_en_mask = 0, ecc_en_mask = 0;
3497 struct amd64_umc *umc;
3498
3499 for_each_umc(i) {
3500 umc = &pvt->umc[i];
3501
3502
3503 if (!(umc->sdp_ctrl & UMC_SDP_INIT))
3504 continue;
3505
3506 umc_en_mask |= BIT(i);
3507
3508 if (umc->umc_cap_hi & UMC_ECC_ENABLED)
3509 ecc_en_mask |= BIT(i);
3510 }
3511
3512
3513 if (umc_en_mask)
3514 ecc_en = umc_en_mask == ecc_en_mask;
3515 else
3516 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3517
3518
3519 nb_mce_en = true;
3520 } else {
3521 amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
3522
3523 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3524
3525 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3526 if (!nb_mce_en)
3527 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3528 MSR_IA32_MCG_CTL, nid);
3529 }
3530
3531 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3532
3533 if (!ecc_en || !nb_mce_en)
3534 return false;
3535 else
3536 return true;
3537}
3538
3539static inline void
3540f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3541{
3542 u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
3543
3544 for_each_umc(i) {
3545 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3546 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3547 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3548
3549 dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3550 dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
3551 }
3552 }
3553
3554
3555 if (ecc_en) {
3556 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3557
3558 if (!cpk_en)
3559 return;
3560
3561 if (dev_x4)
3562 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3563 else if (dev_x16)
3564 mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3565 else
3566 mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
3567 }
3568}
3569
3570static void setup_mci_misc_attrs(struct mem_ctl_info *mci)
3571{
3572 struct amd64_pvt *pvt = mci->pvt_info;
3573
3574 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3575 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3576
3577 if (pvt->umc) {
3578 f17h_determine_edac_ctl_cap(mci, pvt);
3579 } else {
3580 if (pvt->nbcap & NBCAP_SECDED)
3581 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3582
3583 if (pvt->nbcap & NBCAP_CHIPKILL)
3584 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3585 }
3586
3587 mci->edac_cap = determine_edac_cap(pvt);
3588 mci->mod_name = EDAC_MOD_STR;
3589 mci->ctl_name = fam_type->ctl_name;
3590 mci->dev_name = pci_name(pvt->F3);
3591 mci->ctl_page_to_phys = NULL;
3592
3593
3594 mci->set_sdram_scrub_rate = set_scrub_rate;
3595 mci->get_sdram_scrub_rate = get_scrub_rate;
3596}
3597
3598
3599
3600
3601static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3602{
3603 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3604 pvt->stepping = boot_cpu_data.x86_stepping;
3605 pvt->model = boot_cpu_data.x86_model;
3606 pvt->fam = boot_cpu_data.x86;
3607
3608 switch (pvt->fam) {
3609 case 0xf:
3610 fam_type = &family_types[K8_CPUS];
3611 pvt->ops = &family_types[K8_CPUS].ops;
3612 break;
3613
3614 case 0x10:
3615 fam_type = &family_types[F10_CPUS];
3616 pvt->ops = &family_types[F10_CPUS].ops;
3617 break;
3618
3619 case 0x15:
3620 if (pvt->model == 0x30) {
3621 fam_type = &family_types[F15_M30H_CPUS];
3622 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3623 break;
3624 } else if (pvt->model == 0x60) {
3625 fam_type = &family_types[F15_M60H_CPUS];
3626 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3627 break;
3628
3629 } else if (pvt->model == 0x13) {
3630 return NULL;
3631 } else {
3632 fam_type = &family_types[F15_CPUS];
3633 pvt->ops = &family_types[F15_CPUS].ops;
3634 }
3635 break;
3636
3637 case 0x16:
3638 if (pvt->model == 0x30) {
3639 fam_type = &family_types[F16_M30H_CPUS];
3640 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3641 break;
3642 }
3643 fam_type = &family_types[F16_CPUS];
3644 pvt->ops = &family_types[F16_CPUS].ops;
3645 break;
3646
3647 case 0x17:
3648 if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3649 fam_type = &family_types[F17_M10H_CPUS];
3650 pvt->ops = &family_types[F17_M10H_CPUS].ops;
3651 break;
3652 } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3653 fam_type = &family_types[F17_M30H_CPUS];
3654 pvt->ops = &family_types[F17_M30H_CPUS].ops;
3655 break;
3656 } else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
3657 fam_type = &family_types[F17_M60H_CPUS];
3658 pvt->ops = &family_types[F17_M60H_CPUS].ops;
3659 break;
3660 } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
3661 fam_type = &family_types[F17_M70H_CPUS];
3662 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3663 break;
3664 }
3665 fallthrough;
3666 case 0x18:
3667 fam_type = &family_types[F17_CPUS];
3668 pvt->ops = &family_types[F17_CPUS].ops;
3669
3670 if (pvt->fam == 0x18)
3671 family_types[F17_CPUS].ctl_name = "F18h";
3672 break;
3673
3674 case 0x19:
3675 if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
3676 fam_type = &family_types[F17_M70H_CPUS];
3677 pvt->ops = &family_types[F17_M70H_CPUS].ops;
3678 fam_type->ctl_name = "F19h_M20h";
3679 break;
3680 }
3681 fam_type = &family_types[F19_CPUS];
3682 pvt->ops = &family_types[F19_CPUS].ops;
3683 family_types[F19_CPUS].ctl_name = "F19h";
3684 break;
3685
3686 default:
3687 amd64_err("Unsupported family!\n");
3688 return NULL;
3689 }
3690
3691 return fam_type;
3692}
3693
3694static const struct attribute_group *amd64_edac_attr_groups[] = {
3695#ifdef CONFIG_EDAC_DEBUG
3696 &dbg_group,
3697 &inj_group,
3698#endif
3699 NULL
3700};
3701
3702static int hw_info_get(struct amd64_pvt *pvt)
3703{
3704 u16 pci_id1, pci_id2;
3705 int ret;
3706
3707 if (pvt->fam >= 0x17) {
3708 pvt->umc = kcalloc(fam_type->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
3709 if (!pvt->umc)
3710 return -ENOMEM;
3711
3712 pci_id1 = fam_type->f0_id;
3713 pci_id2 = fam_type->f6_id;
3714 } else {
3715 pci_id1 = fam_type->f1_id;
3716 pci_id2 = fam_type->f2_id;
3717 }
3718
3719 ret = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3720 if (ret)
3721 return ret;
3722
3723 read_mc_regs(pvt);
3724
3725 return 0;
3726}
3727
3728static void hw_info_put(struct amd64_pvt *pvt)
3729{
3730 if (pvt->F0 || pvt->F1)
3731 free_mc_sibling_devs(pvt);
3732
3733 kfree(pvt->umc);
3734}
3735
3736static int init_one_instance(struct amd64_pvt *pvt)
3737{
3738 struct mem_ctl_info *mci = NULL;
3739 struct edac_mc_layer layers[2];
3740 int ret = -EINVAL;
3741
3742
3743
3744
3745
3746
3747 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3748 if (pvt->channel_count < 0)
3749 return ret;
3750
3751 ret = -ENOMEM;
3752 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3753 layers[0].size = pvt->csels[0].b_cnt;
3754 layers[0].is_virt_csrow = true;
3755 layers[1].type = EDAC_MC_LAYER_CHANNEL;
3756
3757
3758
3759
3760
3761
3762 layers[1].size = fam_type->max_mcs;
3763 layers[1].is_virt_csrow = false;
3764
3765 mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
3766 if (!mci)
3767 return ret;
3768
3769 mci->pvt_info = pvt;
3770 mci->pdev = &pvt->F3->dev;
3771
3772 setup_mci_misc_attrs(mci);
3773
3774 if (init_csrows(mci))
3775 mci->edac_cap = EDAC_FLAG_NONE;
3776
3777 ret = -ENODEV;
3778 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3779 edac_dbg(1, "failed edac_mc_add_mc()\n");
3780 edac_mc_free(mci);
3781 return ret;
3782 }
3783
3784 return 0;
3785}
3786
3787static bool instance_has_memory(struct amd64_pvt *pvt)
3788{
3789 bool cs_enabled = false;
3790 int cs = 0, dct = 0;
3791
3792 for (dct = 0; dct < fam_type->max_mcs; dct++) {
3793 for_each_chip_select(cs, dct, pvt)
3794 cs_enabled |= csrow_enabled(cs, dct, pvt);
3795 }
3796
3797 return cs_enabled;
3798}
3799
3800static int probe_one_instance(unsigned int nid)
3801{
3802 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3803 struct amd64_pvt *pvt = NULL;
3804 struct ecc_settings *s;
3805 int ret;
3806
3807 ret = -ENOMEM;
3808 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3809 if (!s)
3810 goto err_out;
3811
3812 ecc_stngs[nid] = s;
3813
3814 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3815 if (!pvt)
3816 goto err_settings;
3817
3818 pvt->mc_node_id = nid;
3819 pvt->F3 = F3;
3820
3821 ret = -ENODEV;
3822 fam_type = per_family_init(pvt);
3823 if (!fam_type)
3824 goto err_enable;
3825
3826 ret = hw_info_get(pvt);
3827 if (ret < 0)
3828 goto err_enable;
3829
3830 ret = 0;
3831 if (!instance_has_memory(pvt)) {
3832 amd64_info("Node %d: No DIMMs detected.\n", nid);
3833 goto err_enable;
3834 }
3835
3836 if (!ecc_enabled(pvt)) {
3837 ret = -ENODEV;
3838
3839 if (!ecc_enable_override)
3840 goto err_enable;
3841
3842 if (boot_cpu_data.x86 >= 0x17) {
3843 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3844 goto err_enable;
3845 } else
3846 amd64_warn("Forcing ECC on!\n");
3847
3848 if (!enable_ecc_error_reporting(s, nid, F3))
3849 goto err_enable;
3850 }
3851
3852 ret = init_one_instance(pvt);
3853 if (ret < 0) {
3854 amd64_err("Error probing instance: %d\n", nid);
3855
3856 if (boot_cpu_data.x86 < 0x17)
3857 restore_ecc_error_reporting(s, nid, F3);
3858
3859 goto err_enable;
3860 }
3861
3862 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3863 (pvt->fam == 0xf ?
3864 (pvt->ext_model >= K8_REV_F ? "revF or later "
3865 : "revE or earlier ")
3866 : ""), pvt->mc_node_id);
3867
3868 dump_misc_regs(pvt);
3869
3870 return ret;
3871
3872err_enable:
3873 hw_info_put(pvt);
3874 kfree(pvt);
3875
3876err_settings:
3877 kfree(s);
3878 ecc_stngs[nid] = NULL;
3879
3880err_out:
3881 return ret;
3882}
3883
3884static void remove_one_instance(unsigned int nid)
3885{
3886 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3887 struct ecc_settings *s = ecc_stngs[nid];
3888 struct mem_ctl_info *mci;
3889 struct amd64_pvt *pvt;
3890
3891
3892 mci = edac_mc_del_mc(&F3->dev);
3893 if (!mci)
3894 return;
3895
3896 pvt = mci->pvt_info;
3897
3898 restore_ecc_error_reporting(s, nid, F3);
3899
3900 kfree(ecc_stngs[nid]);
3901 ecc_stngs[nid] = NULL;
3902
3903
3904 mci->pvt_info = NULL;
3905
3906 hw_info_put(pvt);
3907 kfree(pvt);
3908 edac_mc_free(mci);
3909}
3910
3911static void setup_pci_device(void)
3912{
3913 if (pci_ctl)
3914 return;
3915
3916 pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
3917 if (!pci_ctl) {
3918 pr_warn("%s(): Unable to create PCI control\n", __func__);
3919 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3920 }
3921}
3922
3923static const struct x86_cpu_id amd64_cpuids[] = {
3924 X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL),
3925 X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL),
3926 X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL),
3927 X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL),
3928 X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL),
3929 X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL),
3930 X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL),
3931 { }
3932};
3933MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3934
3935static int __init amd64_edac_init(void)
3936{
3937 const char *owner;
3938 int err = -ENODEV;
3939 int i;
3940
3941 owner = edac_get_owner();
3942 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3943 return -EBUSY;
3944
3945 if (!x86_match_cpu(amd64_cpuids))
3946 return -ENODEV;
3947
3948 if (amd_cache_northbridges() < 0)
3949 return -ENODEV;
3950
3951 opstate_init();
3952
3953 err = -ENOMEM;
3954 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
3955 if (!ecc_stngs)
3956 goto err_free;
3957
3958 msrs = msrs_alloc();
3959 if (!msrs)
3960 goto err_free;
3961
3962 for (i = 0; i < amd_nb_num(); i++) {
3963 err = probe_one_instance(i);
3964 if (err) {
3965
3966 while (--i >= 0)
3967 remove_one_instance(i);
3968
3969 goto err_pci;
3970 }
3971 }
3972
3973 if (!edac_has_mcs()) {
3974 err = -ENODEV;
3975 goto err_pci;
3976 }
3977
3978
3979 if (boot_cpu_data.x86 >= 0x17)
3980 amd_register_ecc_decoder(decode_umc_error);
3981 else
3982 amd_register_ecc_decoder(decode_bus_error);
3983
3984 setup_pci_device();
3985
3986#ifdef CONFIG_X86_32
3987 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3988#endif
3989
3990 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3991
3992 return 0;
3993
3994err_pci:
3995 pci_ctl_dev = NULL;
3996
3997 msrs_free(msrs);
3998 msrs = NULL;
3999
4000err_free:
4001 kfree(ecc_stngs);
4002 ecc_stngs = NULL;
4003
4004 return err;
4005}
4006
4007static void __exit amd64_edac_exit(void)
4008{
4009 int i;
4010
4011 if (pci_ctl)
4012 edac_pci_release_generic_ctl(pci_ctl);
4013
4014
4015 if (boot_cpu_data.x86 >= 0x17)
4016 amd_unregister_ecc_decoder(decode_umc_error);
4017 else
4018 amd_unregister_ecc_decoder(decode_bus_error);
4019
4020 for (i = 0; i < amd_nb_num(); i++)
4021 remove_one_instance(i);
4022
4023 kfree(ecc_stngs);
4024 ecc_stngs = NULL;
4025
4026 pci_ctl_dev = NULL;
4027
4028 msrs_free(msrs);
4029 msrs = NULL;
4030}
4031
4032module_init(amd64_edac_init);
4033module_exit(amd64_edac_exit);
4034
4035MODULE_LICENSE("GPL");
4036MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
4037 "Dave Peterson, Thayne Harbaugh");
4038MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
4039 EDAC_AMD64_VERSION);
4040
4041module_param(edac_op_state, int, 0444);
4042MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
4043