linux/drivers/edac/edac_mc.c
<<
>>
Prefs
   1/*
   2 * edac_mc kernel module
   3 * (C) 2005, 2006 Linux Networx (http://lnxi.com)
   4 * This file may be distributed under the terms of the
   5 * GNU General Public License.
   6 *
   7 * Written by Thayne Harbaugh
   8 * Based on work by Dan Hollis <goemon at anime dot net> and others.
   9 *      http://www.anime.net/~goemon/linux-ecc/
  10 *
  11 * Modified by Dave Peterson and Doug Thompson
  12 *
  13 */
  14
  15#include <linux/module.h>
  16#include <linux/proc_fs.h>
  17#include <linux/kernel.h>
  18#include <linux/types.h>
  19#include <linux/smp.h>
  20#include <linux/init.h>
  21#include <linux/sysctl.h>
  22#include <linux/highmem.h>
  23#include <linux/timer.h>
  24#include <linux/slab.h>
  25#include <linux/jiffies.h>
  26#include <linux/spinlock.h>
  27#include <linux/list.h>
  28#include <linux/ctype.h>
  29#include <linux/edac.h>
  30#include <linux/bitops.h>
  31#include <linux/uaccess.h>
  32#include <asm/page.h>
  33#include "edac_mc.h"
  34#include "edac_module.h"
  35#include <ras/ras_event.h>
  36
  37#ifdef CONFIG_EDAC_ATOMIC_SCRUB
  38#include <asm/edac.h>
  39#else
  40#define edac_atomic_scrub(va, size) do { } while (0)
  41#endif
  42
  43int edac_op_state = EDAC_OPSTATE_INVAL;
  44EXPORT_SYMBOL_GPL(edac_op_state);
  45
  46/* lock to memory controller's control array */
  47static DEFINE_MUTEX(mem_ctls_mutex);
  48static LIST_HEAD(mc_devices);
  49
  50/*
  51 * Used to lock EDAC MC to just one module, avoiding two drivers e. g.
  52 *      apei/ghes and i7core_edac to be used at the same time.
  53 */
  54static const char *edac_mc_owner;
  55
  56static struct mem_ctl_info *error_desc_to_mci(struct edac_raw_error_desc *e)
  57{
  58        return container_of(e, struct mem_ctl_info, error_desc);
  59}
  60
  61unsigned int edac_dimm_info_location(struct dimm_info *dimm, char *buf,
  62                                     unsigned int len)
  63{
  64        struct mem_ctl_info *mci = dimm->mci;
  65        int i, n, count = 0;
  66        char *p = buf;
  67
  68        for (i = 0; i < mci->n_layers; i++) {
  69                n = snprintf(p, len, "%s %d ",
  70                              edac_layer_name[mci->layers[i].type],
  71                              dimm->location[i]);
  72                p += n;
  73                len -= n;
  74                count += n;
  75                if (!len)
  76                        break;
  77        }
  78
  79        return count;
  80}
  81
  82#ifdef CONFIG_EDAC_DEBUG
  83
  84static void edac_mc_dump_channel(struct rank_info *chan)
  85{
  86        edac_dbg(4, "  channel->chan_idx = %d\n", chan->chan_idx);
  87        edac_dbg(4, "    channel = %p\n", chan);
  88        edac_dbg(4, "    channel->csrow = %p\n", chan->csrow);
  89        edac_dbg(4, "    channel->dimm = %p\n", chan->dimm);
  90}
  91
  92static void edac_mc_dump_dimm(struct dimm_info *dimm)
  93{
  94        char location[80];
  95
  96        if (!dimm->nr_pages)
  97                return;
  98
  99        edac_dimm_info_location(dimm, location, sizeof(location));
 100
 101        edac_dbg(4, "%s%i: %smapped as virtual row %d, chan %d\n",
 102                 dimm->mci->csbased ? "rank" : "dimm",
 103                 dimm->idx, location, dimm->csrow, dimm->cschannel);
 104        edac_dbg(4, "  dimm = %p\n", dimm);
 105        edac_dbg(4, "  dimm->label = '%s'\n", dimm->label);
 106        edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
 107        edac_dbg(4, "  dimm->grain = %d\n", dimm->grain);
 108        edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
 109}
 110
 111static void edac_mc_dump_csrow(struct csrow_info *csrow)
 112{
 113        edac_dbg(4, "csrow->csrow_idx = %d\n", csrow->csrow_idx);
 114        edac_dbg(4, "  csrow = %p\n", csrow);
 115        edac_dbg(4, "  csrow->first_page = 0x%lx\n", csrow->first_page);
 116        edac_dbg(4, "  csrow->last_page = 0x%lx\n", csrow->last_page);
 117        edac_dbg(4, "  csrow->page_mask = 0x%lx\n", csrow->page_mask);
 118        edac_dbg(4, "  csrow->nr_channels = %d\n", csrow->nr_channels);
 119        edac_dbg(4, "  csrow->channels = %p\n", csrow->channels);
 120        edac_dbg(4, "  csrow->mci = %p\n", csrow->mci);
 121}
 122
 123static void edac_mc_dump_mci(struct mem_ctl_info *mci)
 124{
 125        edac_dbg(3, "\tmci = %p\n", mci);
 126        edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);
 127        edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);
 128        edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);
 129        edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);
 130        edac_dbg(3, "\tmci->nr_csrows = %d, csrows = %p\n",
 131                 mci->nr_csrows, mci->csrows);
 132        edac_dbg(3, "\tmci->nr_dimms = %d, dimms = %p\n",
 133                 mci->tot_dimms, mci->dimms);
 134        edac_dbg(3, "\tdev = %p\n", mci->pdev);
 135        edac_dbg(3, "\tmod_name:ctl_name = %s:%s\n",
 136                 mci->mod_name, mci->ctl_name);
 137        edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);
 138}
 139
 140#endif                          /* CONFIG_EDAC_DEBUG */
 141
 142const char * const edac_mem_types[] = {
 143        [MEM_EMPTY]     = "Empty",
 144        [MEM_RESERVED]  = "Reserved",
 145        [MEM_UNKNOWN]   = "Unknown",
 146        [MEM_FPM]       = "FPM",
 147        [MEM_EDO]       = "EDO",
 148        [MEM_BEDO]      = "BEDO",
 149        [MEM_SDR]       = "Unbuffered-SDR",
 150        [MEM_RDR]       = "Registered-SDR",
 151        [MEM_DDR]       = "Unbuffered-DDR",
 152        [MEM_RDDR]      = "Registered-DDR",
 153        [MEM_RMBS]      = "RMBS",
 154        [MEM_DDR2]      = "Unbuffered-DDR2",
 155        [MEM_FB_DDR2]   = "FullyBuffered-DDR2",
 156        [MEM_RDDR2]     = "Registered-DDR2",
 157        [MEM_XDR]       = "XDR",
 158        [MEM_DDR3]      = "Unbuffered-DDR3",
 159        [MEM_RDDR3]     = "Registered-DDR3",
 160        [MEM_LRDDR3]    = "Load-Reduced-DDR3-RAM",
 161        [MEM_LPDDR3]    = "Low-Power-DDR3-RAM",
 162        [MEM_DDR4]      = "Unbuffered-DDR4",
 163        [MEM_RDDR4]     = "Registered-DDR4",
 164        [MEM_LPDDR4]    = "Low-Power-DDR4-RAM",
 165        [MEM_LRDDR4]    = "Load-Reduced-DDR4-RAM",
 166        [MEM_DDR5]      = "Unbuffered-DDR5",
 167        [MEM_NVDIMM]    = "Non-volatile-RAM",
 168        [MEM_WIO2]      = "Wide-IO-2",
 169};
 170EXPORT_SYMBOL_GPL(edac_mem_types);
 171
 172/**
 173 * edac_align_ptr - Prepares the pointer offsets for a single-shot allocation
 174 * @p:          pointer to a pointer with the memory offset to be used. At
 175 *              return, this will be incremented to point to the next offset
 176 * @size:       Size of the data structure to be reserved
 177 * @n_elems:    Number of elements that should be reserved
 178 *
 179 * If 'size' is a constant, the compiler will optimize this whole function
 180 * down to either a no-op or the addition of a constant to the value of '*p'.
 181 *
 182 * The 'p' pointer is absolutely needed to keep the proper advancing
 183 * further in memory to the proper offsets when allocating the struct along
 184 * with its embedded structs, as edac_device_alloc_ctl_info() does it
 185 * above, for example.
 186 *
 187 * At return, the pointer 'p' will be incremented to be used on a next call
 188 * to this function.
 189 */
 190void *edac_align_ptr(void **p, unsigned int size, int n_elems)
 191{
 192        unsigned int align, r;
 193        void *ptr = *p;
 194
 195        *p += size * n_elems;
 196
 197        /*
 198         * 'p' can possibly be an unaligned item X such that sizeof(X) is
 199         * 'size'.  Adjust 'p' so that its alignment is at least as
 200         * stringent as what the compiler would provide for X and return
 201         * the aligned result.
 202         * Here we assume that the alignment of a "long long" is the most
 203         * stringent alignment that the compiler will ever provide by default.
 204         * As far as I know, this is a reasonable assumption.
 205         */
 206        if (size > sizeof(long))
 207                align = sizeof(long long);
 208        else if (size > sizeof(int))
 209                align = sizeof(long);
 210        else if (size > sizeof(short))
 211                align = sizeof(int);
 212        else if (size > sizeof(char))
 213                align = sizeof(short);
 214        else
 215                return (char *)ptr;
 216
 217        r = (unsigned long)p % align;
 218
 219        if (r == 0)
 220                return (char *)ptr;
 221
 222        *p += align - r;
 223
 224        return (void *)(((unsigned long)ptr) + align - r);
 225}
 226
 227static void _edac_mc_free(struct mem_ctl_info *mci)
 228{
 229        put_device(&mci->dev);
 230}
 231
 232static void mci_release(struct device *dev)
 233{
 234        struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
 235        struct csrow_info *csr;
 236        int i, chn, row;
 237
 238        if (mci->dimms) {
 239                for (i = 0; i < mci->tot_dimms; i++)
 240                        kfree(mci->dimms[i]);
 241                kfree(mci->dimms);
 242        }
 243
 244        if (mci->csrows) {
 245                for (row = 0; row < mci->nr_csrows; row++) {
 246                        csr = mci->csrows[row];
 247                        if (!csr)
 248                                continue;
 249
 250                        if (csr->channels) {
 251                                for (chn = 0; chn < mci->num_cschannel; chn++)
 252                                        kfree(csr->channels[chn]);
 253                                kfree(csr->channels);
 254                        }
 255                        kfree(csr);
 256                }
 257                kfree(mci->csrows);
 258        }
 259        kfree(mci);
 260}
 261
 262static int edac_mc_alloc_csrows(struct mem_ctl_info *mci)
 263{
 264        unsigned int tot_channels = mci->num_cschannel;
 265        unsigned int tot_csrows = mci->nr_csrows;
 266        unsigned int row, chn;
 267
 268        /*
 269         * Alocate and fill the csrow/channels structs
 270         */
 271        mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);
 272        if (!mci->csrows)
 273                return -ENOMEM;
 274
 275        for (row = 0; row < tot_csrows; row++) {
 276                struct csrow_info *csr;
 277
 278                csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);
 279                if (!csr)
 280                        return -ENOMEM;
 281
 282                mci->csrows[row] = csr;
 283                csr->csrow_idx = row;
 284                csr->mci = mci;
 285                csr->nr_channels = tot_channels;
 286                csr->channels = kcalloc(tot_channels, sizeof(*csr->channels),
 287                                        GFP_KERNEL);
 288                if (!csr->channels)
 289                        return -ENOMEM;
 290
 291                for (chn = 0; chn < tot_channels; chn++) {
 292                        struct rank_info *chan;
 293
 294                        chan = kzalloc(sizeof(**csr->channels), GFP_KERNEL);
 295                        if (!chan)
 296                                return -ENOMEM;
 297
 298                        csr->channels[chn] = chan;
 299                        chan->chan_idx = chn;
 300                        chan->csrow = csr;
 301                }
 302        }
 303
 304        return 0;
 305}
 306
 307static int edac_mc_alloc_dimms(struct mem_ctl_info *mci)
 308{
 309        unsigned int pos[EDAC_MAX_LAYERS];
 310        unsigned int row, chn, idx;
 311        int layer;
 312        void *p;
 313
 314        /*
 315         * Allocate and fill the dimm structs
 316         */
 317        mci->dimms  = kcalloc(mci->tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);
 318        if (!mci->dimms)
 319                return -ENOMEM;
 320
 321        memset(&pos, 0, sizeof(pos));
 322        row = 0;
 323        chn = 0;
 324        for (idx = 0; idx < mci->tot_dimms; idx++) {
 325                struct dimm_info *dimm;
 326                struct rank_info *chan;
 327                int n, len;
 328
 329                chan = mci->csrows[row]->channels[chn];
 330
 331                dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);
 332                if (!dimm)
 333                        return -ENOMEM;
 334                mci->dimms[idx] = dimm;
 335                dimm->mci = mci;
 336                dimm->idx = idx;
 337
 338                /*
 339                 * Copy DIMM location and initialize it.
 340                 */
 341                len = sizeof(dimm->label);
 342                p = dimm->label;
 343                n = snprintf(p, len, "mc#%u", mci->mc_idx);
 344                p += n;
 345                len -= n;
 346                for (layer = 0; layer < mci->n_layers; layer++) {
 347                        n = snprintf(p, len, "%s#%u",
 348                                     edac_layer_name[mci->layers[layer].type],
 349                                     pos[layer]);
 350                        p += n;
 351                        len -= n;
 352                        dimm->location[layer] = pos[layer];
 353
 354                        if (len <= 0)
 355                                break;
 356                }
 357
 358                /* Link it to the csrows old API data */
 359                chan->dimm = dimm;
 360                dimm->csrow = row;
 361                dimm->cschannel = chn;
 362
 363                /* Increment csrow location */
 364                if (mci->layers[0].is_virt_csrow) {
 365                        chn++;
 366                        if (chn == mci->num_cschannel) {
 367                                chn = 0;
 368                                row++;
 369                        }
 370                } else {
 371                        row++;
 372                        if (row == mci->nr_csrows) {
 373                                row = 0;
 374                                chn++;
 375                        }
 376                }
 377
 378                /* Increment dimm location */
 379                for (layer = mci->n_layers - 1; layer >= 0; layer--) {
 380                        pos[layer]++;
 381                        if (pos[layer] < mci->layers[layer].size)
 382                                break;
 383                        pos[layer] = 0;
 384                }
 385        }
 386
 387        return 0;
 388}
 389
 390struct mem_ctl_info *edac_mc_alloc(unsigned int mc_num,
 391                                   unsigned int n_layers,
 392                                   struct edac_mc_layer *layers,
 393                                   unsigned int sz_pvt)
 394{
 395        struct mem_ctl_info *mci;
 396        struct edac_mc_layer *layer;
 397        unsigned int idx, size, tot_dimms = 1;
 398        unsigned int tot_csrows = 1, tot_channels = 1;
 399        void *pvt, *ptr = NULL;
 400        bool per_rank = false;
 401
 402        if (WARN_ON(n_layers > EDAC_MAX_LAYERS || n_layers == 0))
 403                return NULL;
 404
 405        /*
 406         * Calculate the total amount of dimms and csrows/cschannels while
 407         * in the old API emulation mode
 408         */
 409        for (idx = 0; idx < n_layers; idx++) {
 410                tot_dimms *= layers[idx].size;
 411
 412                if (layers[idx].is_virt_csrow)
 413                        tot_csrows *= layers[idx].size;
 414                else
 415                        tot_channels *= layers[idx].size;
 416
 417                if (layers[idx].type == EDAC_MC_LAYER_CHIP_SELECT)
 418                        per_rank = true;
 419        }
 420
 421        /* Figure out the offsets of the various items from the start of an mc
 422         * structure.  We want the alignment of each item to be at least as
 423         * stringent as what the compiler would provide if we could simply
 424         * hardcode everything into a single struct.
 425         */
 426        mci     = edac_align_ptr(&ptr, sizeof(*mci), 1);
 427        layer   = edac_align_ptr(&ptr, sizeof(*layer), n_layers);
 428        pvt     = edac_align_ptr(&ptr, sz_pvt, 1);
 429        size    = ((unsigned long)pvt) + sz_pvt;
 430
 431        edac_dbg(1, "allocating %u bytes for mci data (%d %s, %d csrows/channels)\n",
 432                 size,
 433                 tot_dimms,
 434                 per_rank ? "ranks" : "dimms",
 435                 tot_csrows * tot_channels);
 436
 437        mci = kzalloc(size, GFP_KERNEL);
 438        if (mci == NULL)
 439                return NULL;
 440
 441        mci->dev.release = mci_release;
 442        device_initialize(&mci->dev);
 443
 444        /* Adjust pointers so they point within the memory we just allocated
 445         * rather than an imaginary chunk of memory located at address 0.
 446         */
 447        layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));
 448        pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;
 449
 450        /* setup index and various internal pointers */
 451        mci->mc_idx = mc_num;
 452        mci->tot_dimms = tot_dimms;
 453        mci->pvt_info = pvt;
 454        mci->n_layers = n_layers;
 455        mci->layers = layer;
 456        memcpy(mci->layers, layers, sizeof(*layer) * n_layers);
 457        mci->nr_csrows = tot_csrows;
 458        mci->num_cschannel = tot_channels;
 459        mci->csbased = per_rank;
 460
 461        if (edac_mc_alloc_csrows(mci))
 462                goto error;
 463
 464        if (edac_mc_alloc_dimms(mci))
 465                goto error;
 466
 467        mci->op_state = OP_ALLOC;
 468
 469        return mci;
 470
 471error:
 472        _edac_mc_free(mci);
 473
 474        return NULL;
 475}
 476EXPORT_SYMBOL_GPL(edac_mc_alloc);
 477
 478void edac_mc_free(struct mem_ctl_info *mci)
 479{
 480        edac_dbg(1, "\n");
 481
 482        _edac_mc_free(mci);
 483}
 484EXPORT_SYMBOL_GPL(edac_mc_free);
 485
 486bool edac_has_mcs(void)
 487{
 488        bool ret;
 489
 490        mutex_lock(&mem_ctls_mutex);
 491
 492        ret = list_empty(&mc_devices);
 493
 494        mutex_unlock(&mem_ctls_mutex);
 495
 496        return !ret;
 497}
 498EXPORT_SYMBOL_GPL(edac_has_mcs);
 499
 500/* Caller must hold mem_ctls_mutex */
 501static struct mem_ctl_info *__find_mci_by_dev(struct device *dev)
 502{
 503        struct mem_ctl_info *mci;
 504        struct list_head *item;
 505
 506        edac_dbg(3, "\n");
 507
 508        list_for_each(item, &mc_devices) {
 509                mci = list_entry(item, struct mem_ctl_info, link);
 510
 511                if (mci->pdev == dev)
 512                        return mci;
 513        }
 514
 515        return NULL;
 516}
 517
 518/**
 519 * find_mci_by_dev
 520 *
 521 *      scan list of controllers looking for the one that manages
 522 *      the 'dev' device
 523 * @dev: pointer to a struct device related with the MCI
 524 */
 525struct mem_ctl_info *find_mci_by_dev(struct device *dev)
 526{
 527        struct mem_ctl_info *ret;
 528
 529        mutex_lock(&mem_ctls_mutex);
 530        ret = __find_mci_by_dev(dev);
 531        mutex_unlock(&mem_ctls_mutex);
 532
 533        return ret;
 534}
 535EXPORT_SYMBOL_GPL(find_mci_by_dev);
 536
 537/*
 538 * edac_mc_workq_function
 539 *      performs the operation scheduled by a workq request
 540 */
 541static void edac_mc_workq_function(struct work_struct *work_req)
 542{
 543        struct delayed_work *d_work = to_delayed_work(work_req);
 544        struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);
 545
 546        mutex_lock(&mem_ctls_mutex);
 547
 548        if (mci->op_state != OP_RUNNING_POLL) {
 549                mutex_unlock(&mem_ctls_mutex);
 550                return;
 551        }
 552
 553        if (edac_op_state == EDAC_OPSTATE_POLL)
 554                mci->edac_check(mci);
 555
 556        mutex_unlock(&mem_ctls_mutex);
 557
 558        /* Queue ourselves again. */
 559        edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
 560}
 561
 562/*
 563 * edac_mc_reset_delay_period(unsigned long value)
 564 *
 565 *      user space has updated our poll period value, need to
 566 *      reset our workq delays
 567 */
 568void edac_mc_reset_delay_period(unsigned long value)
 569{
 570        struct mem_ctl_info *mci;
 571        struct list_head *item;
 572
 573        mutex_lock(&mem_ctls_mutex);
 574
 575        list_for_each(item, &mc_devices) {
 576                mci = list_entry(item, struct mem_ctl_info, link);
 577
 578                if (mci->op_state == OP_RUNNING_POLL)
 579                        edac_mod_work(&mci->work, value);
 580        }
 581        mutex_unlock(&mem_ctls_mutex);
 582}
 583
 584
 585
 586/* Return 0 on success, 1 on failure.
 587 * Before calling this function, caller must
 588 * assign a unique value to mci->mc_idx.
 589 *
 590 *      locking model:
 591 *
 592 *              called with the mem_ctls_mutex lock held
 593 */
 594static int add_mc_to_global_list(struct mem_ctl_info *mci)
 595{
 596        struct list_head *item, *insert_before;
 597        struct mem_ctl_info *p;
 598
 599        insert_before = &mc_devices;
 600
 601        p = __find_mci_by_dev(mci->pdev);
 602        if (unlikely(p != NULL))
 603                goto fail0;
 604
 605        list_for_each(item, &mc_devices) {
 606                p = list_entry(item, struct mem_ctl_info, link);
 607
 608                if (p->mc_idx >= mci->mc_idx) {
 609                        if (unlikely(p->mc_idx == mci->mc_idx))
 610                                goto fail1;
 611
 612                        insert_before = item;
 613                        break;
 614                }
 615        }
 616
 617        list_add_tail_rcu(&mci->link, insert_before);
 618        return 0;
 619
 620fail0:
 621        edac_printk(KERN_WARNING, EDAC_MC,
 622                "%s (%s) %s %s already assigned %d\n", dev_name(p->pdev),
 623                edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);
 624        return 1;
 625
 626fail1:
 627        edac_printk(KERN_WARNING, EDAC_MC,
 628                "bug in low-level driver: attempt to assign\n"
 629                "    duplicate mc_idx %d in %s()\n", p->mc_idx, __func__);
 630        return 1;
 631}
 632
 633static int del_mc_from_global_list(struct mem_ctl_info *mci)
 634{
 635        list_del_rcu(&mci->link);
 636
 637        /* these are for safe removal of devices from global list while
 638         * NMI handlers may be traversing list
 639         */
 640        synchronize_rcu();
 641        INIT_LIST_HEAD(&mci->link);
 642
 643        return list_empty(&mc_devices);
 644}
 645
 646struct mem_ctl_info *edac_mc_find(int idx)
 647{
 648        struct mem_ctl_info *mci;
 649        struct list_head *item;
 650
 651        mutex_lock(&mem_ctls_mutex);
 652
 653        list_for_each(item, &mc_devices) {
 654                mci = list_entry(item, struct mem_ctl_info, link);
 655                if (mci->mc_idx == idx)
 656                        goto unlock;
 657        }
 658
 659        mci = NULL;
 660unlock:
 661        mutex_unlock(&mem_ctls_mutex);
 662        return mci;
 663}
 664EXPORT_SYMBOL(edac_mc_find);
 665
 666const char *edac_get_owner(void)
 667{
 668        return edac_mc_owner;
 669}
 670EXPORT_SYMBOL_GPL(edac_get_owner);
 671
 672/* FIXME - should a warning be printed if no error detection? correction? */
 673int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
 674                               const struct attribute_group **groups)
 675{
 676        int ret = -EINVAL;
 677        edac_dbg(0, "\n");
 678
 679#ifdef CONFIG_EDAC_DEBUG
 680        if (edac_debug_level >= 3)
 681                edac_mc_dump_mci(mci);
 682
 683        if (edac_debug_level >= 4) {
 684                struct dimm_info *dimm;
 685                int i;
 686
 687                for (i = 0; i < mci->nr_csrows; i++) {
 688                        struct csrow_info *csrow = mci->csrows[i];
 689                        u32 nr_pages = 0;
 690                        int j;
 691
 692                        for (j = 0; j < csrow->nr_channels; j++)
 693                                nr_pages += csrow->channels[j]->dimm->nr_pages;
 694                        if (!nr_pages)
 695                                continue;
 696                        edac_mc_dump_csrow(csrow);
 697                        for (j = 0; j < csrow->nr_channels; j++)
 698                                if (csrow->channels[j]->dimm->nr_pages)
 699                                        edac_mc_dump_channel(csrow->channels[j]);
 700                }
 701
 702                mci_for_each_dimm(mci, dimm)
 703                        edac_mc_dump_dimm(dimm);
 704        }
 705#endif
 706        mutex_lock(&mem_ctls_mutex);
 707
 708        if (edac_mc_owner && edac_mc_owner != mci->mod_name) {
 709                ret = -EPERM;
 710                goto fail0;
 711        }
 712
 713        if (add_mc_to_global_list(mci))
 714                goto fail0;
 715
 716        /* set load time so that error rate can be tracked */
 717        mci->start_time = jiffies;
 718
 719        mci->bus = edac_get_sysfs_subsys();
 720
 721        if (edac_create_sysfs_mci_device(mci, groups)) {
 722                edac_mc_printk(mci, KERN_WARNING,
 723                        "failed to create sysfs device\n");
 724                goto fail1;
 725        }
 726
 727        if (mci->edac_check) {
 728                mci->op_state = OP_RUNNING_POLL;
 729
 730                INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);
 731                edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
 732
 733        } else {
 734                mci->op_state = OP_RUNNING_INTERRUPT;
 735        }
 736
 737        /* Report action taken */
 738        edac_mc_printk(mci, KERN_INFO,
 739                "Giving out device to module %s controller %s: DEV %s (%s)\n",
 740                mci->mod_name, mci->ctl_name, mci->dev_name,
 741                edac_op_state_to_string(mci->op_state));
 742
 743        edac_mc_owner = mci->mod_name;
 744
 745        mutex_unlock(&mem_ctls_mutex);
 746        return 0;
 747
 748fail1:
 749        del_mc_from_global_list(mci);
 750
 751fail0:
 752        mutex_unlock(&mem_ctls_mutex);
 753        return ret;
 754}
 755EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups);
 756
 757struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
 758{
 759        struct mem_ctl_info *mci;
 760
 761        edac_dbg(0, "\n");
 762
 763        mutex_lock(&mem_ctls_mutex);
 764
 765        /* find the requested mci struct in the global list */
 766        mci = __find_mci_by_dev(dev);
 767        if (mci == NULL) {
 768                mutex_unlock(&mem_ctls_mutex);
 769                return NULL;
 770        }
 771
 772        /* mark MCI offline: */
 773        mci->op_state = OP_OFFLINE;
 774
 775        if (del_mc_from_global_list(mci))
 776                edac_mc_owner = NULL;
 777
 778        mutex_unlock(&mem_ctls_mutex);
 779
 780        if (mci->edac_check)
 781                edac_stop_work(&mci->work);
 782
 783        /* remove from sysfs */
 784        edac_remove_sysfs_mci_device(mci);
 785
 786        edac_printk(KERN_INFO, EDAC_MC,
 787                "Removed device %d for %s %s: DEV %s\n", mci->mc_idx,
 788                mci->mod_name, mci->ctl_name, edac_dev_name(mci));
 789
 790        return mci;
 791}
 792EXPORT_SYMBOL_GPL(edac_mc_del_mc);
 793
 794static void edac_mc_scrub_block(unsigned long page, unsigned long offset,
 795                                u32 size)
 796{
 797        struct page *pg;
 798        void *virt_addr;
 799        unsigned long flags = 0;
 800
 801        edac_dbg(3, "\n");
 802
 803        /* ECC error page was not in our memory. Ignore it. */
 804        if (!pfn_valid(page))
 805                return;
 806
 807        /* Find the actual page structure then map it and fix */
 808        pg = pfn_to_page(page);
 809
 810        if (PageHighMem(pg))
 811                local_irq_save(flags);
 812
 813        virt_addr = kmap_atomic(pg);
 814
 815        /* Perform architecture specific atomic scrub operation */
 816        edac_atomic_scrub(virt_addr + offset, size);
 817
 818        /* Unmap and complete */
 819        kunmap_atomic(virt_addr);
 820
 821        if (PageHighMem(pg))
 822                local_irq_restore(flags);
 823}
 824
 825/* FIXME - should return -1 */
 826int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)
 827{
 828        struct csrow_info **csrows = mci->csrows;
 829        int row, i, j, n;
 830
 831        edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);
 832        row = -1;
 833
 834        for (i = 0; i < mci->nr_csrows; i++) {
 835                struct csrow_info *csrow = csrows[i];
 836                n = 0;
 837                for (j = 0; j < csrow->nr_channels; j++) {
 838                        struct dimm_info *dimm = csrow->channels[j]->dimm;
 839                        n += dimm->nr_pages;
 840                }
 841                if (n == 0)
 842                        continue;
 843
 844                edac_dbg(3, "MC%d: first(0x%lx) page(0x%lx) last(0x%lx) mask(0x%lx)\n",
 845                         mci->mc_idx,
 846                         csrow->first_page, page, csrow->last_page,
 847                         csrow->page_mask);
 848
 849                if ((page >= csrow->first_page) &&
 850                    (page <= csrow->last_page) &&
 851                    ((page & csrow->page_mask) ==
 852                     (csrow->first_page & csrow->page_mask))) {
 853                        row = i;
 854                        break;
 855                }
 856        }
 857
 858        if (row == -1)
 859                edac_mc_printk(mci, KERN_ERR,
 860                        "could not look up page error address %lx\n",
 861                        (unsigned long)page);
 862
 863        return row;
 864}
 865EXPORT_SYMBOL_GPL(edac_mc_find_csrow_by_page);
 866
 867const char *edac_layer_name[] = {
 868        [EDAC_MC_LAYER_BRANCH] = "branch",
 869        [EDAC_MC_LAYER_CHANNEL] = "channel",
 870        [EDAC_MC_LAYER_SLOT] = "slot",
 871        [EDAC_MC_LAYER_CHIP_SELECT] = "csrow",
 872        [EDAC_MC_LAYER_ALL_MEM] = "memory",
 873};
 874EXPORT_SYMBOL_GPL(edac_layer_name);
 875
 876static void edac_inc_ce_error(struct edac_raw_error_desc *e)
 877{
 878        int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };
 879        struct mem_ctl_info *mci = error_desc_to_mci(e);
 880        struct dimm_info *dimm = edac_get_dimm(mci, pos[0], pos[1], pos[2]);
 881
 882        mci->ce_mc += e->error_count;
 883
 884        if (dimm)
 885                dimm->ce_count += e->error_count;
 886        else
 887                mci->ce_noinfo_count += e->error_count;
 888}
 889
 890static void edac_inc_ue_error(struct edac_raw_error_desc *e)
 891{
 892        int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };
 893        struct mem_ctl_info *mci = error_desc_to_mci(e);
 894        struct dimm_info *dimm = edac_get_dimm(mci, pos[0], pos[1], pos[2]);
 895
 896        mci->ue_mc += e->error_count;
 897
 898        if (dimm)
 899                dimm->ue_count += e->error_count;
 900        else
 901                mci->ue_noinfo_count += e->error_count;
 902}
 903
 904static void edac_ce_error(struct edac_raw_error_desc *e)
 905{
 906        struct mem_ctl_info *mci = error_desc_to_mci(e);
 907        unsigned long remapped_page;
 908
 909        if (edac_mc_get_log_ce()) {
 910                edac_mc_printk(mci, KERN_WARNING,
 911                        "%d CE %s%son %s (%s page:0x%lx offset:0x%lx grain:%ld syndrome:0x%lx%s%s)\n",
 912                        e->error_count, e->msg,
 913                        *e->msg ? " " : "",
 914                        e->label, e->location, e->page_frame_number, e->offset_in_page,
 915                        e->grain, e->syndrome,
 916                        *e->other_detail ? " - " : "",
 917                        e->other_detail);
 918        }
 919
 920        edac_inc_ce_error(e);
 921
 922        if (mci->scrub_mode == SCRUB_SW_SRC) {
 923                /*
 924                        * Some memory controllers (called MCs below) can remap
 925                        * memory so that it is still available at a different
 926                        * address when PCI devices map into memory.
 927                        * MC's that can't do this, lose the memory where PCI
 928                        * devices are mapped. This mapping is MC-dependent
 929                        * and so we call back into the MC driver for it to
 930                        * map the MC page to a physical (CPU) page which can
 931                        * then be mapped to a virtual page - which can then
 932                        * be scrubbed.
 933                        */
 934                remapped_page = mci->ctl_page_to_phys ?
 935                        mci->ctl_page_to_phys(mci, e->page_frame_number) :
 936                        e->page_frame_number;
 937
 938                edac_mc_scrub_block(remapped_page, e->offset_in_page, e->grain);
 939        }
 940}
 941
 942static void edac_ue_error(struct edac_raw_error_desc *e)
 943{
 944        struct mem_ctl_info *mci = error_desc_to_mci(e);
 945
 946        if (edac_mc_get_log_ue()) {
 947                edac_mc_printk(mci, KERN_WARNING,
 948                        "%d UE %s%son %s (%s page:0x%lx offset:0x%lx grain:%ld%s%s)\n",
 949                        e->error_count, e->msg,
 950                        *e->msg ? " " : "",
 951                        e->label, e->location, e->page_frame_number, e->offset_in_page,
 952                        e->grain,
 953                        *e->other_detail ? " - " : "",
 954                        e->other_detail);
 955        }
 956
 957        edac_inc_ue_error(e);
 958
 959        if (edac_mc_get_panic_on_ue()) {
 960                panic("UE %s%son %s (%s page:0x%lx offset:0x%lx grain:%ld%s%s)\n",
 961                        e->msg,
 962                        *e->msg ? " " : "",
 963                        e->label, e->location, e->page_frame_number, e->offset_in_page,
 964                        e->grain,
 965                        *e->other_detail ? " - " : "",
 966                        e->other_detail);
 967        }
 968}
 969
 970static void edac_inc_csrow(struct edac_raw_error_desc *e, int row, int chan)
 971{
 972        struct mem_ctl_info *mci = error_desc_to_mci(e);
 973        enum hw_event_mc_err_type type = e->type;
 974        u16 count = e->error_count;
 975
 976        if (row < 0)
 977                return;
 978
 979        edac_dbg(4, "csrow/channel to increment: (%d,%d)\n", row, chan);
 980
 981        if (type == HW_EVENT_ERR_CORRECTED) {
 982                mci->csrows[row]->ce_count += count;
 983                if (chan >= 0)
 984                        mci->csrows[row]->channels[chan]->ce_count += count;
 985        } else {
 986                mci->csrows[row]->ue_count += count;
 987        }
 988}
 989
 990void edac_raw_mc_handle_error(struct edac_raw_error_desc *e)
 991{
 992        struct mem_ctl_info *mci = error_desc_to_mci(e);
 993        u8 grain_bits;
 994
 995        /* Sanity-check driver-supplied grain value. */
 996        if (WARN_ON_ONCE(!e->grain))
 997                e->grain = 1;
 998
 999        grain_bits = fls_long(e->grain - 1);
1000
1001        /* Report the error via the trace interface */
1002        if (IS_ENABLED(CONFIG_RAS))
1003                trace_mc_event(e->type, e->msg, e->label, e->error_count,
1004                               mci->mc_idx, e->top_layer, e->mid_layer,
1005                               e->low_layer,
1006                               (e->page_frame_number << PAGE_SHIFT) | e->offset_in_page,
1007                               grain_bits, e->syndrome, e->other_detail);
1008
1009        if (e->type == HW_EVENT_ERR_CORRECTED)
1010                edac_ce_error(e);
1011        else
1012                edac_ue_error(e);
1013}
1014EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error);
1015
1016void edac_mc_handle_error(const enum hw_event_mc_err_type type,
1017                          struct mem_ctl_info *mci,
1018                          const u16 error_count,
1019                          const unsigned long page_frame_number,
1020                          const unsigned long offset_in_page,
1021                          const unsigned long syndrome,
1022                          const int top_layer,
1023                          const int mid_layer,
1024                          const int low_layer,
1025                          const char *msg,
1026                          const char *other_detail)
1027{
1028        struct dimm_info *dimm;
1029        char *p;
1030        int row = -1, chan = -1;
1031        int pos[EDAC_MAX_LAYERS] = { top_layer, mid_layer, low_layer };
1032        int i, n_labels = 0;
1033        struct edac_raw_error_desc *e = &mci->error_desc;
1034        bool any_memory = true;
1035
1036        edac_dbg(3, "MC%d\n", mci->mc_idx);
1037
1038        /* Fills the error report buffer */
1039        memset(e, 0, sizeof (*e));
1040        e->error_count = error_count;
1041        e->type = type;
1042        e->top_layer = top_layer;
1043        e->mid_layer = mid_layer;
1044        e->low_layer = low_layer;
1045        e->page_frame_number = page_frame_number;
1046        e->offset_in_page = offset_in_page;
1047        e->syndrome = syndrome;
1048        /* need valid strings here for both: */
1049        e->msg = msg ?: "";
1050        e->other_detail = other_detail ?: "";
1051
1052        /*
1053         * Check if the event report is consistent and if the memory location is
1054         * known. If it is, the DIMM(s) label info will be filled and the DIMM's
1055         * error counters will be incremented.
1056         */
1057        for (i = 0; i < mci->n_layers; i++) {
1058                if (pos[i] >= (int)mci->layers[i].size) {
1059
1060                        edac_mc_printk(mci, KERN_ERR,
1061                                       "INTERNAL ERROR: %s value is out of range (%d >= %d)\n",
1062                                       edac_layer_name[mci->layers[i].type],
1063                                       pos[i], mci->layers[i].size);
1064                        /*
1065                         * Instead of just returning it, let's use what's
1066                         * known about the error. The increment routines and
1067                         * the DIMM filter logic will do the right thing by
1068                         * pointing the likely damaged DIMMs.
1069                         */
1070                        pos[i] = -1;
1071                }
1072                if (pos[i] >= 0)
1073                        any_memory = false;
1074        }
1075
1076        /*
1077         * Get the dimm label/grain that applies to the match criteria.
1078         * As the error algorithm may not be able to point to just one memory
1079         * stick, the logic here will get all possible labels that could
1080         * pottentially be affected by the error.
1081         * On FB-DIMM memory controllers, for uncorrected errors, it is common
1082         * to have only the MC channel and the MC dimm (also called "branch")
1083         * but the channel is not known, as the memory is arranged in pairs,
1084         * where each memory belongs to a separate channel within the same
1085         * branch.
1086         */
1087        p = e->label;
1088        *p = '\0';
1089
1090        mci_for_each_dimm(mci, dimm) {
1091                if (top_layer >= 0 && top_layer != dimm->location[0])
1092                        continue;
1093                if (mid_layer >= 0 && mid_layer != dimm->location[1])
1094                        continue;
1095                if (low_layer >= 0 && low_layer != dimm->location[2])
1096                        continue;
1097
1098                /* get the max grain, over the error match range */
1099                if (dimm->grain > e->grain)
1100                        e->grain = dimm->grain;
1101
1102                /*
1103                 * If the error is memory-controller wide, there's no need to
1104                 * seek for the affected DIMMs because the whole channel/memory
1105                 * controller/... may be affected. Also, don't show errors for
1106                 * empty DIMM slots.
1107                 */
1108                if (!dimm->nr_pages)
1109                        continue;
1110
1111                n_labels++;
1112                if (n_labels > EDAC_MAX_LABELS) {
1113                        p = e->label;
1114                        *p = '\0';
1115                } else {
1116                        if (p != e->label) {
1117                                strcpy(p, OTHER_LABEL);
1118                                p += strlen(OTHER_LABEL);
1119                        }
1120                        strcpy(p, dimm->label);
1121                        p += strlen(p);
1122                }
1123
1124                /*
1125                 * get csrow/channel of the DIMM, in order to allow
1126                 * incrementing the compat API counters
1127                 */
1128                edac_dbg(4, "%s csrows map: (%d,%d)\n",
1129                        mci->csbased ? "rank" : "dimm",
1130                        dimm->csrow, dimm->cschannel);
1131                if (row == -1)
1132                        row = dimm->csrow;
1133                else if (row >= 0 && row != dimm->csrow)
1134                        row = -2;
1135
1136                if (chan == -1)
1137                        chan = dimm->cschannel;
1138                else if (chan >= 0 && chan != dimm->cschannel)
1139                        chan = -2;
1140        }
1141
1142        if (any_memory)
1143                strcpy(e->label, "any memory");
1144        else if (!*e->label)
1145                strcpy(e->label, "unknown memory");
1146
1147        edac_inc_csrow(e, row, chan);
1148
1149        /* Fill the RAM location data */
1150        p = e->location;
1151
1152        for (i = 0; i < mci->n_layers; i++) {
1153                if (pos[i] < 0)
1154                        continue;
1155
1156                p += sprintf(p, "%s:%d ",
1157                             edac_layer_name[mci->layers[i].type],
1158                             pos[i]);
1159        }
1160        if (p > e->location)
1161                *(p - 1) = '\0';
1162
1163        edac_raw_mc_handle_error(e);
1164}
1165EXPORT_SYMBOL_GPL(edac_mc_handle_error);
1166