1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/module.h>
16#include <linux/proc_fs.h>
17#include <linux/kernel.h>
18#include <linux/types.h>
19#include <linux/smp.h>
20#include <linux/init.h>
21#include <linux/sysctl.h>
22#include <linux/highmem.h>
23#include <linux/timer.h>
24#include <linux/slab.h>
25#include <linux/jiffies.h>
26#include <linux/spinlock.h>
27#include <linux/list.h>
28#include <linux/ctype.h>
29#include <linux/edac.h>
30#include <linux/bitops.h>
31#include <linux/uaccess.h>
32#include <asm/page.h>
33#include "edac_mc.h"
34#include "edac_module.h"
35#include <ras/ras_event.h>
36
37#ifdef CONFIG_EDAC_ATOMIC_SCRUB
38#include <asm/edac.h>
39#else
40#define edac_atomic_scrub(va, size) do { } while (0)
41#endif
42
43int edac_op_state = EDAC_OPSTATE_INVAL;
44EXPORT_SYMBOL_GPL(edac_op_state);
45
46
47static DEFINE_MUTEX(mem_ctls_mutex);
48static LIST_HEAD(mc_devices);
49
50
51
52
53
54static const char *edac_mc_owner;
55
56static struct mem_ctl_info *error_desc_to_mci(struct edac_raw_error_desc *e)
57{
58 return container_of(e, struct mem_ctl_info, error_desc);
59}
60
61unsigned int edac_dimm_info_location(struct dimm_info *dimm, char *buf,
62 unsigned int len)
63{
64 struct mem_ctl_info *mci = dimm->mci;
65 int i, n, count = 0;
66 char *p = buf;
67
68 for (i = 0; i < mci->n_layers; i++) {
69 n = snprintf(p, len, "%s %d ",
70 edac_layer_name[mci->layers[i].type],
71 dimm->location[i]);
72 p += n;
73 len -= n;
74 count += n;
75 if (!len)
76 break;
77 }
78
79 return count;
80}
81
82#ifdef CONFIG_EDAC_DEBUG
83
84static void edac_mc_dump_channel(struct rank_info *chan)
85{
86 edac_dbg(4, " channel->chan_idx = %d\n", chan->chan_idx);
87 edac_dbg(4, " channel = %p\n", chan);
88 edac_dbg(4, " channel->csrow = %p\n", chan->csrow);
89 edac_dbg(4, " channel->dimm = %p\n", chan->dimm);
90}
91
92static void edac_mc_dump_dimm(struct dimm_info *dimm)
93{
94 char location[80];
95
96 if (!dimm->nr_pages)
97 return;
98
99 edac_dimm_info_location(dimm, location, sizeof(location));
100
101 edac_dbg(4, "%s%i: %smapped as virtual row %d, chan %d\n",
102 dimm->mci->csbased ? "rank" : "dimm",
103 dimm->idx, location, dimm->csrow, dimm->cschannel);
104 edac_dbg(4, " dimm = %p\n", dimm);
105 edac_dbg(4, " dimm->label = '%s'\n", dimm->label);
106 edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm->nr_pages);
107 edac_dbg(4, " dimm->grain = %d\n", dimm->grain);
108 edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm->nr_pages);
109}
110
111static void edac_mc_dump_csrow(struct csrow_info *csrow)
112{
113 edac_dbg(4, "csrow->csrow_idx = %d\n", csrow->csrow_idx);
114 edac_dbg(4, " csrow = %p\n", csrow);
115 edac_dbg(4, " csrow->first_page = 0x%lx\n", csrow->first_page);
116 edac_dbg(4, " csrow->last_page = 0x%lx\n", csrow->last_page);
117 edac_dbg(4, " csrow->page_mask = 0x%lx\n", csrow->page_mask);
118 edac_dbg(4, " csrow->nr_channels = %d\n", csrow->nr_channels);
119 edac_dbg(4, " csrow->channels = %p\n", csrow->channels);
120 edac_dbg(4, " csrow->mci = %p\n", csrow->mci);
121}
122
123static void edac_mc_dump_mci(struct mem_ctl_info *mci)
124{
125 edac_dbg(3, "\tmci = %p\n", mci);
126 edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);
127 edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);
128 edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);
129 edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);
130 edac_dbg(3, "\tmci->nr_csrows = %d, csrows = %p\n",
131 mci->nr_csrows, mci->csrows);
132 edac_dbg(3, "\tmci->nr_dimms = %d, dimms = %p\n",
133 mci->tot_dimms, mci->dimms);
134 edac_dbg(3, "\tdev = %p\n", mci->pdev);
135 edac_dbg(3, "\tmod_name:ctl_name = %s:%s\n",
136 mci->mod_name, mci->ctl_name);
137 edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);
138}
139
140#endif
141
142const char * const edac_mem_types[] = {
143 [MEM_EMPTY] = "Empty",
144 [MEM_RESERVED] = "Reserved",
145 [MEM_UNKNOWN] = "Unknown",
146 [MEM_FPM] = "FPM",
147 [MEM_EDO] = "EDO",
148 [MEM_BEDO] = "BEDO",
149 [MEM_SDR] = "Unbuffered-SDR",
150 [MEM_RDR] = "Registered-SDR",
151 [MEM_DDR] = "Unbuffered-DDR",
152 [MEM_RDDR] = "Registered-DDR",
153 [MEM_RMBS] = "RMBS",
154 [MEM_DDR2] = "Unbuffered-DDR2",
155 [MEM_FB_DDR2] = "FullyBuffered-DDR2",
156 [MEM_RDDR2] = "Registered-DDR2",
157 [MEM_XDR] = "XDR",
158 [MEM_DDR3] = "Unbuffered-DDR3",
159 [MEM_RDDR3] = "Registered-DDR3",
160 [MEM_LRDDR3] = "Load-Reduced-DDR3-RAM",
161 [MEM_LPDDR3] = "Low-Power-DDR3-RAM",
162 [MEM_DDR4] = "Unbuffered-DDR4",
163 [MEM_RDDR4] = "Registered-DDR4",
164 [MEM_LPDDR4] = "Low-Power-DDR4-RAM",
165 [MEM_LRDDR4] = "Load-Reduced-DDR4-RAM",
166 [MEM_DDR5] = "Unbuffered-DDR5",
167 [MEM_NVDIMM] = "Non-volatile-RAM",
168 [MEM_WIO2] = "Wide-IO-2",
169};
170EXPORT_SYMBOL_GPL(edac_mem_types);
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190void *edac_align_ptr(void **p, unsigned int size, int n_elems)
191{
192 unsigned int align, r;
193 void *ptr = *p;
194
195 *p += size * n_elems;
196
197
198
199
200
201
202
203
204
205
206 if (size > sizeof(long))
207 align = sizeof(long long);
208 else if (size > sizeof(int))
209 align = sizeof(long);
210 else if (size > sizeof(short))
211 align = sizeof(int);
212 else if (size > sizeof(char))
213 align = sizeof(short);
214 else
215 return (char *)ptr;
216
217 r = (unsigned long)p % align;
218
219 if (r == 0)
220 return (char *)ptr;
221
222 *p += align - r;
223
224 return (void *)(((unsigned long)ptr) + align - r);
225}
226
227static void _edac_mc_free(struct mem_ctl_info *mci)
228{
229 put_device(&mci->dev);
230}
231
232static void mci_release(struct device *dev)
233{
234 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
235 struct csrow_info *csr;
236 int i, chn, row;
237
238 if (mci->dimms) {
239 for (i = 0; i < mci->tot_dimms; i++)
240 kfree(mci->dimms[i]);
241 kfree(mci->dimms);
242 }
243
244 if (mci->csrows) {
245 for (row = 0; row < mci->nr_csrows; row++) {
246 csr = mci->csrows[row];
247 if (!csr)
248 continue;
249
250 if (csr->channels) {
251 for (chn = 0; chn < mci->num_cschannel; chn++)
252 kfree(csr->channels[chn]);
253 kfree(csr->channels);
254 }
255 kfree(csr);
256 }
257 kfree(mci->csrows);
258 }
259 kfree(mci);
260}
261
262static int edac_mc_alloc_csrows(struct mem_ctl_info *mci)
263{
264 unsigned int tot_channels = mci->num_cschannel;
265 unsigned int tot_csrows = mci->nr_csrows;
266 unsigned int row, chn;
267
268
269
270
271 mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);
272 if (!mci->csrows)
273 return -ENOMEM;
274
275 for (row = 0; row < tot_csrows; row++) {
276 struct csrow_info *csr;
277
278 csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);
279 if (!csr)
280 return -ENOMEM;
281
282 mci->csrows[row] = csr;
283 csr->csrow_idx = row;
284 csr->mci = mci;
285 csr->nr_channels = tot_channels;
286 csr->channels = kcalloc(tot_channels, sizeof(*csr->channels),
287 GFP_KERNEL);
288 if (!csr->channels)
289 return -ENOMEM;
290
291 for (chn = 0; chn < tot_channels; chn++) {
292 struct rank_info *chan;
293
294 chan = kzalloc(sizeof(**csr->channels), GFP_KERNEL);
295 if (!chan)
296 return -ENOMEM;
297
298 csr->channels[chn] = chan;
299 chan->chan_idx = chn;
300 chan->csrow = csr;
301 }
302 }
303
304 return 0;
305}
306
307static int edac_mc_alloc_dimms(struct mem_ctl_info *mci)
308{
309 unsigned int pos[EDAC_MAX_LAYERS];
310 unsigned int row, chn, idx;
311 int layer;
312 void *p;
313
314
315
316
317 mci->dimms = kcalloc(mci->tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);
318 if (!mci->dimms)
319 return -ENOMEM;
320
321 memset(&pos, 0, sizeof(pos));
322 row = 0;
323 chn = 0;
324 for (idx = 0; idx < mci->tot_dimms; idx++) {
325 struct dimm_info *dimm;
326 struct rank_info *chan;
327 int n, len;
328
329 chan = mci->csrows[row]->channels[chn];
330
331 dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);
332 if (!dimm)
333 return -ENOMEM;
334 mci->dimms[idx] = dimm;
335 dimm->mci = mci;
336 dimm->idx = idx;
337
338
339
340
341 len = sizeof(dimm->label);
342 p = dimm->label;
343 n = snprintf(p, len, "mc#%u", mci->mc_idx);
344 p += n;
345 len -= n;
346 for (layer = 0; layer < mci->n_layers; layer++) {
347 n = snprintf(p, len, "%s#%u",
348 edac_layer_name[mci->layers[layer].type],
349 pos[layer]);
350 p += n;
351 len -= n;
352 dimm->location[layer] = pos[layer];
353
354 if (len <= 0)
355 break;
356 }
357
358
359 chan->dimm = dimm;
360 dimm->csrow = row;
361 dimm->cschannel = chn;
362
363
364 if (mci->layers[0].is_virt_csrow) {
365 chn++;
366 if (chn == mci->num_cschannel) {
367 chn = 0;
368 row++;
369 }
370 } else {
371 row++;
372 if (row == mci->nr_csrows) {
373 row = 0;
374 chn++;
375 }
376 }
377
378
379 for (layer = mci->n_layers - 1; layer >= 0; layer--) {
380 pos[layer]++;
381 if (pos[layer] < mci->layers[layer].size)
382 break;
383 pos[layer] = 0;
384 }
385 }
386
387 return 0;
388}
389
390struct mem_ctl_info *edac_mc_alloc(unsigned int mc_num,
391 unsigned int n_layers,
392 struct edac_mc_layer *layers,
393 unsigned int sz_pvt)
394{
395 struct mem_ctl_info *mci;
396 struct edac_mc_layer *layer;
397 unsigned int idx, size, tot_dimms = 1;
398 unsigned int tot_csrows = 1, tot_channels = 1;
399 void *pvt, *ptr = NULL;
400 bool per_rank = false;
401
402 if (WARN_ON(n_layers > EDAC_MAX_LAYERS || n_layers == 0))
403 return NULL;
404
405
406
407
408
409 for (idx = 0; idx < n_layers; idx++) {
410 tot_dimms *= layers[idx].size;
411
412 if (layers[idx].is_virt_csrow)
413 tot_csrows *= layers[idx].size;
414 else
415 tot_channels *= layers[idx].size;
416
417 if (layers[idx].type == EDAC_MC_LAYER_CHIP_SELECT)
418 per_rank = true;
419 }
420
421
422
423
424
425
426 mci = edac_align_ptr(&ptr, sizeof(*mci), 1);
427 layer = edac_align_ptr(&ptr, sizeof(*layer), n_layers);
428 pvt = edac_align_ptr(&ptr, sz_pvt, 1);
429 size = ((unsigned long)pvt) + sz_pvt;
430
431 edac_dbg(1, "allocating %u bytes for mci data (%d %s, %d csrows/channels)\n",
432 size,
433 tot_dimms,
434 per_rank ? "ranks" : "dimms",
435 tot_csrows * tot_channels);
436
437 mci = kzalloc(size, GFP_KERNEL);
438 if (mci == NULL)
439 return NULL;
440
441 mci->dev.release = mci_release;
442 device_initialize(&mci->dev);
443
444
445
446
447 layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));
448 pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;
449
450
451 mci->mc_idx = mc_num;
452 mci->tot_dimms = tot_dimms;
453 mci->pvt_info = pvt;
454 mci->n_layers = n_layers;
455 mci->layers = layer;
456 memcpy(mci->layers, layers, sizeof(*layer) * n_layers);
457 mci->nr_csrows = tot_csrows;
458 mci->num_cschannel = tot_channels;
459 mci->csbased = per_rank;
460
461 if (edac_mc_alloc_csrows(mci))
462 goto error;
463
464 if (edac_mc_alloc_dimms(mci))
465 goto error;
466
467 mci->op_state = OP_ALLOC;
468
469 return mci;
470
471error:
472 _edac_mc_free(mci);
473
474 return NULL;
475}
476EXPORT_SYMBOL_GPL(edac_mc_alloc);
477
478void edac_mc_free(struct mem_ctl_info *mci)
479{
480 edac_dbg(1, "\n");
481
482 _edac_mc_free(mci);
483}
484EXPORT_SYMBOL_GPL(edac_mc_free);
485
486bool edac_has_mcs(void)
487{
488 bool ret;
489
490 mutex_lock(&mem_ctls_mutex);
491
492 ret = list_empty(&mc_devices);
493
494 mutex_unlock(&mem_ctls_mutex);
495
496 return !ret;
497}
498EXPORT_SYMBOL_GPL(edac_has_mcs);
499
500
501static struct mem_ctl_info *__find_mci_by_dev(struct device *dev)
502{
503 struct mem_ctl_info *mci;
504 struct list_head *item;
505
506 edac_dbg(3, "\n");
507
508 list_for_each(item, &mc_devices) {
509 mci = list_entry(item, struct mem_ctl_info, link);
510
511 if (mci->pdev == dev)
512 return mci;
513 }
514
515 return NULL;
516}
517
518
519
520
521
522
523
524
525struct mem_ctl_info *find_mci_by_dev(struct device *dev)
526{
527 struct mem_ctl_info *ret;
528
529 mutex_lock(&mem_ctls_mutex);
530 ret = __find_mci_by_dev(dev);
531 mutex_unlock(&mem_ctls_mutex);
532
533 return ret;
534}
535EXPORT_SYMBOL_GPL(find_mci_by_dev);
536
537
538
539
540
541static void edac_mc_workq_function(struct work_struct *work_req)
542{
543 struct delayed_work *d_work = to_delayed_work(work_req);
544 struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);
545
546 mutex_lock(&mem_ctls_mutex);
547
548 if (mci->op_state != OP_RUNNING_POLL) {
549 mutex_unlock(&mem_ctls_mutex);
550 return;
551 }
552
553 if (edac_op_state == EDAC_OPSTATE_POLL)
554 mci->edac_check(mci);
555
556 mutex_unlock(&mem_ctls_mutex);
557
558
559 edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
560}
561
562
563
564
565
566
567
568void edac_mc_reset_delay_period(unsigned long value)
569{
570 struct mem_ctl_info *mci;
571 struct list_head *item;
572
573 mutex_lock(&mem_ctls_mutex);
574
575 list_for_each(item, &mc_devices) {
576 mci = list_entry(item, struct mem_ctl_info, link);
577
578 if (mci->op_state == OP_RUNNING_POLL)
579 edac_mod_work(&mci->work, value);
580 }
581 mutex_unlock(&mem_ctls_mutex);
582}
583
584
585
586
587
588
589
590
591
592
593
594static int add_mc_to_global_list(struct mem_ctl_info *mci)
595{
596 struct list_head *item, *insert_before;
597 struct mem_ctl_info *p;
598
599 insert_before = &mc_devices;
600
601 p = __find_mci_by_dev(mci->pdev);
602 if (unlikely(p != NULL))
603 goto fail0;
604
605 list_for_each(item, &mc_devices) {
606 p = list_entry(item, struct mem_ctl_info, link);
607
608 if (p->mc_idx >= mci->mc_idx) {
609 if (unlikely(p->mc_idx == mci->mc_idx))
610 goto fail1;
611
612 insert_before = item;
613 break;
614 }
615 }
616
617 list_add_tail_rcu(&mci->link, insert_before);
618 return 0;
619
620fail0:
621 edac_printk(KERN_WARNING, EDAC_MC,
622 "%s (%s) %s %s already assigned %d\n", dev_name(p->pdev),
623 edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);
624 return 1;
625
626fail1:
627 edac_printk(KERN_WARNING, EDAC_MC,
628 "bug in low-level driver: attempt to assign\n"
629 " duplicate mc_idx %d in %s()\n", p->mc_idx, __func__);
630 return 1;
631}
632
633static int del_mc_from_global_list(struct mem_ctl_info *mci)
634{
635 list_del_rcu(&mci->link);
636
637
638
639
640 synchronize_rcu();
641 INIT_LIST_HEAD(&mci->link);
642
643 return list_empty(&mc_devices);
644}
645
646struct mem_ctl_info *edac_mc_find(int idx)
647{
648 struct mem_ctl_info *mci;
649 struct list_head *item;
650
651 mutex_lock(&mem_ctls_mutex);
652
653 list_for_each(item, &mc_devices) {
654 mci = list_entry(item, struct mem_ctl_info, link);
655 if (mci->mc_idx == idx)
656 goto unlock;
657 }
658
659 mci = NULL;
660unlock:
661 mutex_unlock(&mem_ctls_mutex);
662 return mci;
663}
664EXPORT_SYMBOL(edac_mc_find);
665
666const char *edac_get_owner(void)
667{
668 return edac_mc_owner;
669}
670EXPORT_SYMBOL_GPL(edac_get_owner);
671
672
673int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
674 const struct attribute_group **groups)
675{
676 int ret = -EINVAL;
677 edac_dbg(0, "\n");
678
679#ifdef CONFIG_EDAC_DEBUG
680 if (edac_debug_level >= 3)
681 edac_mc_dump_mci(mci);
682
683 if (edac_debug_level >= 4) {
684 struct dimm_info *dimm;
685 int i;
686
687 for (i = 0; i < mci->nr_csrows; i++) {
688 struct csrow_info *csrow = mci->csrows[i];
689 u32 nr_pages = 0;
690 int j;
691
692 for (j = 0; j < csrow->nr_channels; j++)
693 nr_pages += csrow->channels[j]->dimm->nr_pages;
694 if (!nr_pages)
695 continue;
696 edac_mc_dump_csrow(csrow);
697 for (j = 0; j < csrow->nr_channels; j++)
698 if (csrow->channels[j]->dimm->nr_pages)
699 edac_mc_dump_channel(csrow->channels[j]);
700 }
701
702 mci_for_each_dimm(mci, dimm)
703 edac_mc_dump_dimm(dimm);
704 }
705#endif
706 mutex_lock(&mem_ctls_mutex);
707
708 if (edac_mc_owner && edac_mc_owner != mci->mod_name) {
709 ret = -EPERM;
710 goto fail0;
711 }
712
713 if (add_mc_to_global_list(mci))
714 goto fail0;
715
716
717 mci->start_time = jiffies;
718
719 mci->bus = edac_get_sysfs_subsys();
720
721 if (edac_create_sysfs_mci_device(mci, groups)) {
722 edac_mc_printk(mci, KERN_WARNING,
723 "failed to create sysfs device\n");
724 goto fail1;
725 }
726
727 if (mci->edac_check) {
728 mci->op_state = OP_RUNNING_POLL;
729
730 INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);
731 edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
732
733 } else {
734 mci->op_state = OP_RUNNING_INTERRUPT;
735 }
736
737
738 edac_mc_printk(mci, KERN_INFO,
739 "Giving out device to module %s controller %s: DEV %s (%s)\n",
740 mci->mod_name, mci->ctl_name, mci->dev_name,
741 edac_op_state_to_string(mci->op_state));
742
743 edac_mc_owner = mci->mod_name;
744
745 mutex_unlock(&mem_ctls_mutex);
746 return 0;
747
748fail1:
749 del_mc_from_global_list(mci);
750
751fail0:
752 mutex_unlock(&mem_ctls_mutex);
753 return ret;
754}
755EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups);
756
757struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
758{
759 struct mem_ctl_info *mci;
760
761 edac_dbg(0, "\n");
762
763 mutex_lock(&mem_ctls_mutex);
764
765
766 mci = __find_mci_by_dev(dev);
767 if (mci == NULL) {
768 mutex_unlock(&mem_ctls_mutex);
769 return NULL;
770 }
771
772
773 mci->op_state = OP_OFFLINE;
774
775 if (del_mc_from_global_list(mci))
776 edac_mc_owner = NULL;
777
778 mutex_unlock(&mem_ctls_mutex);
779
780 if (mci->edac_check)
781 edac_stop_work(&mci->work);
782
783
784 edac_remove_sysfs_mci_device(mci);
785
786 edac_printk(KERN_INFO, EDAC_MC,
787 "Removed device %d for %s %s: DEV %s\n", mci->mc_idx,
788 mci->mod_name, mci->ctl_name, edac_dev_name(mci));
789
790 return mci;
791}
792EXPORT_SYMBOL_GPL(edac_mc_del_mc);
793
794static void edac_mc_scrub_block(unsigned long page, unsigned long offset,
795 u32 size)
796{
797 struct page *pg;
798 void *virt_addr;
799 unsigned long flags = 0;
800
801 edac_dbg(3, "\n");
802
803
804 if (!pfn_valid(page))
805 return;
806
807
808 pg = pfn_to_page(page);
809
810 if (PageHighMem(pg))
811 local_irq_save(flags);
812
813 virt_addr = kmap_atomic(pg);
814
815
816 edac_atomic_scrub(virt_addr + offset, size);
817
818
819 kunmap_atomic(virt_addr);
820
821 if (PageHighMem(pg))
822 local_irq_restore(flags);
823}
824
825
826int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)
827{
828 struct csrow_info **csrows = mci->csrows;
829 int row, i, j, n;
830
831 edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);
832 row = -1;
833
834 for (i = 0; i < mci->nr_csrows; i++) {
835 struct csrow_info *csrow = csrows[i];
836 n = 0;
837 for (j = 0; j < csrow->nr_channels; j++) {
838 struct dimm_info *dimm = csrow->channels[j]->dimm;
839 n += dimm->nr_pages;
840 }
841 if (n == 0)
842 continue;
843
844 edac_dbg(3, "MC%d: first(0x%lx) page(0x%lx) last(0x%lx) mask(0x%lx)\n",
845 mci->mc_idx,
846 csrow->first_page, page, csrow->last_page,
847 csrow->page_mask);
848
849 if ((page >= csrow->first_page) &&
850 (page <= csrow->last_page) &&
851 ((page & csrow->page_mask) ==
852 (csrow->first_page & csrow->page_mask))) {
853 row = i;
854 break;
855 }
856 }
857
858 if (row == -1)
859 edac_mc_printk(mci, KERN_ERR,
860 "could not look up page error address %lx\n",
861 (unsigned long)page);
862
863 return row;
864}
865EXPORT_SYMBOL_GPL(edac_mc_find_csrow_by_page);
866
867const char *edac_layer_name[] = {
868 [EDAC_MC_LAYER_BRANCH] = "branch",
869 [EDAC_MC_LAYER_CHANNEL] = "channel",
870 [EDAC_MC_LAYER_SLOT] = "slot",
871 [EDAC_MC_LAYER_CHIP_SELECT] = "csrow",
872 [EDAC_MC_LAYER_ALL_MEM] = "memory",
873};
874EXPORT_SYMBOL_GPL(edac_layer_name);
875
876static void edac_inc_ce_error(struct edac_raw_error_desc *e)
877{
878 int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };
879 struct mem_ctl_info *mci = error_desc_to_mci(e);
880 struct dimm_info *dimm = edac_get_dimm(mci, pos[0], pos[1], pos[2]);
881
882 mci->ce_mc += e->error_count;
883
884 if (dimm)
885 dimm->ce_count += e->error_count;
886 else
887 mci->ce_noinfo_count += e->error_count;
888}
889
890static void edac_inc_ue_error(struct edac_raw_error_desc *e)
891{
892 int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };
893 struct mem_ctl_info *mci = error_desc_to_mci(e);
894 struct dimm_info *dimm = edac_get_dimm(mci, pos[0], pos[1], pos[2]);
895
896 mci->ue_mc += e->error_count;
897
898 if (dimm)
899 dimm->ue_count += e->error_count;
900 else
901 mci->ue_noinfo_count += e->error_count;
902}
903
904static void edac_ce_error(struct edac_raw_error_desc *e)
905{
906 struct mem_ctl_info *mci = error_desc_to_mci(e);
907 unsigned long remapped_page;
908
909 if (edac_mc_get_log_ce()) {
910 edac_mc_printk(mci, KERN_WARNING,
911 "%d CE %s%son %s (%s page:0x%lx offset:0x%lx grain:%ld syndrome:0x%lx%s%s)\n",
912 e->error_count, e->msg,
913 *e->msg ? " " : "",
914 e->label, e->location, e->page_frame_number, e->offset_in_page,
915 e->grain, e->syndrome,
916 *e->other_detail ? " - " : "",
917 e->other_detail);
918 }
919
920 edac_inc_ce_error(e);
921
922 if (mci->scrub_mode == SCRUB_SW_SRC) {
923
924
925
926
927
928
929
930
931
932
933
934 remapped_page = mci->ctl_page_to_phys ?
935 mci->ctl_page_to_phys(mci, e->page_frame_number) :
936 e->page_frame_number;
937
938 edac_mc_scrub_block(remapped_page, e->offset_in_page, e->grain);
939 }
940}
941
942static void edac_ue_error(struct edac_raw_error_desc *e)
943{
944 struct mem_ctl_info *mci = error_desc_to_mci(e);
945
946 if (edac_mc_get_log_ue()) {
947 edac_mc_printk(mci, KERN_WARNING,
948 "%d UE %s%son %s (%s page:0x%lx offset:0x%lx grain:%ld%s%s)\n",
949 e->error_count, e->msg,
950 *e->msg ? " " : "",
951 e->label, e->location, e->page_frame_number, e->offset_in_page,
952 e->grain,
953 *e->other_detail ? " - " : "",
954 e->other_detail);
955 }
956
957 edac_inc_ue_error(e);
958
959 if (edac_mc_get_panic_on_ue()) {
960 panic("UE %s%son %s (%s page:0x%lx offset:0x%lx grain:%ld%s%s)\n",
961 e->msg,
962 *e->msg ? " " : "",
963 e->label, e->location, e->page_frame_number, e->offset_in_page,
964 e->grain,
965 *e->other_detail ? " - " : "",
966 e->other_detail);
967 }
968}
969
970static void edac_inc_csrow(struct edac_raw_error_desc *e, int row, int chan)
971{
972 struct mem_ctl_info *mci = error_desc_to_mci(e);
973 enum hw_event_mc_err_type type = e->type;
974 u16 count = e->error_count;
975
976 if (row < 0)
977 return;
978
979 edac_dbg(4, "csrow/channel to increment: (%d,%d)\n", row, chan);
980
981 if (type == HW_EVENT_ERR_CORRECTED) {
982 mci->csrows[row]->ce_count += count;
983 if (chan >= 0)
984 mci->csrows[row]->channels[chan]->ce_count += count;
985 } else {
986 mci->csrows[row]->ue_count += count;
987 }
988}
989
990void edac_raw_mc_handle_error(struct edac_raw_error_desc *e)
991{
992 struct mem_ctl_info *mci = error_desc_to_mci(e);
993 u8 grain_bits;
994
995
996 if (WARN_ON_ONCE(!e->grain))
997 e->grain = 1;
998
999 grain_bits = fls_long(e->grain - 1);
1000
1001
1002 if (IS_ENABLED(CONFIG_RAS))
1003 trace_mc_event(e->type, e->msg, e->label, e->error_count,
1004 mci->mc_idx, e->top_layer, e->mid_layer,
1005 e->low_layer,
1006 (e->page_frame_number << PAGE_SHIFT) | e->offset_in_page,
1007 grain_bits, e->syndrome, e->other_detail);
1008
1009 if (e->type == HW_EVENT_ERR_CORRECTED)
1010 edac_ce_error(e);
1011 else
1012 edac_ue_error(e);
1013}
1014EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error);
1015
1016void edac_mc_handle_error(const enum hw_event_mc_err_type type,
1017 struct mem_ctl_info *mci,
1018 const u16 error_count,
1019 const unsigned long page_frame_number,
1020 const unsigned long offset_in_page,
1021 const unsigned long syndrome,
1022 const int top_layer,
1023 const int mid_layer,
1024 const int low_layer,
1025 const char *msg,
1026 const char *other_detail)
1027{
1028 struct dimm_info *dimm;
1029 char *p;
1030 int row = -1, chan = -1;
1031 int pos[EDAC_MAX_LAYERS] = { top_layer, mid_layer, low_layer };
1032 int i, n_labels = 0;
1033 struct edac_raw_error_desc *e = &mci->error_desc;
1034 bool any_memory = true;
1035
1036 edac_dbg(3, "MC%d\n", mci->mc_idx);
1037
1038
1039 memset(e, 0, sizeof (*e));
1040 e->error_count = error_count;
1041 e->type = type;
1042 e->top_layer = top_layer;
1043 e->mid_layer = mid_layer;
1044 e->low_layer = low_layer;
1045 e->page_frame_number = page_frame_number;
1046 e->offset_in_page = offset_in_page;
1047 e->syndrome = syndrome;
1048
1049 e->msg = msg ?: "";
1050 e->other_detail = other_detail ?: "";
1051
1052
1053
1054
1055
1056
1057 for (i = 0; i < mci->n_layers; i++) {
1058 if (pos[i] >= (int)mci->layers[i].size) {
1059
1060 edac_mc_printk(mci, KERN_ERR,
1061 "INTERNAL ERROR: %s value is out of range (%d >= %d)\n",
1062 edac_layer_name[mci->layers[i].type],
1063 pos[i], mci->layers[i].size);
1064
1065
1066
1067
1068
1069
1070 pos[i] = -1;
1071 }
1072 if (pos[i] >= 0)
1073 any_memory = false;
1074 }
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087 p = e->label;
1088 *p = '\0';
1089
1090 mci_for_each_dimm(mci, dimm) {
1091 if (top_layer >= 0 && top_layer != dimm->location[0])
1092 continue;
1093 if (mid_layer >= 0 && mid_layer != dimm->location[1])
1094 continue;
1095 if (low_layer >= 0 && low_layer != dimm->location[2])
1096 continue;
1097
1098
1099 if (dimm->grain > e->grain)
1100 e->grain = dimm->grain;
1101
1102
1103
1104
1105
1106
1107
1108 if (!dimm->nr_pages)
1109 continue;
1110
1111 n_labels++;
1112 if (n_labels > EDAC_MAX_LABELS) {
1113 p = e->label;
1114 *p = '\0';
1115 } else {
1116 if (p != e->label) {
1117 strcpy(p, OTHER_LABEL);
1118 p += strlen(OTHER_LABEL);
1119 }
1120 strcpy(p, dimm->label);
1121 p += strlen(p);
1122 }
1123
1124
1125
1126
1127
1128 edac_dbg(4, "%s csrows map: (%d,%d)\n",
1129 mci->csbased ? "rank" : "dimm",
1130 dimm->csrow, dimm->cschannel);
1131 if (row == -1)
1132 row = dimm->csrow;
1133 else if (row >= 0 && row != dimm->csrow)
1134 row = -2;
1135
1136 if (chan == -1)
1137 chan = dimm->cschannel;
1138 else if (chan >= 0 && chan != dimm->cschannel)
1139 chan = -2;
1140 }
1141
1142 if (any_memory)
1143 strcpy(e->label, "any memory");
1144 else if (!*e->label)
1145 strcpy(e->label, "unknown memory");
1146
1147 edac_inc_csrow(e, row, chan);
1148
1149
1150 p = e->location;
1151
1152 for (i = 0; i < mci->n_layers; i++) {
1153 if (pos[i] < 0)
1154 continue;
1155
1156 p += sprintf(p, "%s:%d ",
1157 edac_layer_name[mci->layers[i].type],
1158 pos[i]);
1159 }
1160 if (p > e->location)
1161 *(p - 1) = '\0';
1162
1163 edac_raw_mc_handle_error(e);
1164}
1165EXPORT_SYMBOL_GPL(edac_mc_handle_error);
1166