linux/arch/x86/events/amd/uncore.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2013 Advanced Micro Devices, Inc.
   4 *
   5 * Author: Jacob Shin <jacob.shin@amd.com>
   6 */
   7
   8#include <linux/perf_event.h>
   9#include <linux/percpu.h>
  10#include <linux/types.h>
  11#include <linux/slab.h>
  12#include <linux/init.h>
  13#include <linux/cpu.h>
  14#include <linux/cpumask.h>
  15
  16#include <asm/cpufeature.h>
  17#include <asm/perf_event.h>
  18#include <asm/msr.h>
  19#include <asm/smp.h>
  20
  21#define NUM_COUNTERS_NB         4
  22#define NUM_COUNTERS_L2         4
  23#define NUM_COUNTERS_L3         6
  24#define MAX_COUNTERS            6
  25
  26#define RDPMC_BASE_NB           6
  27#define RDPMC_BASE_LLC          10
  28
  29#define COUNTER_SHIFT           16
  30
  31#undef pr_fmt
  32#define pr_fmt(fmt)     "amd_uncore: " fmt
  33
  34static int num_counters_llc;
  35static int num_counters_nb;
  36static bool l3_mask;
  37
  38static HLIST_HEAD(uncore_unused_list);
  39
  40struct amd_uncore {
  41        int id;
  42        int refcnt;
  43        int cpu;
  44        int num_counters;
  45        int rdpmc_base;
  46        u32 msr_base;
  47        cpumask_t *active_mask;
  48        struct pmu *pmu;
  49        struct perf_event *events[MAX_COUNTERS];
  50        struct hlist_node node;
  51};
  52
  53static struct amd_uncore * __percpu *amd_uncore_nb;
  54static struct amd_uncore * __percpu *amd_uncore_llc;
  55
  56static struct pmu amd_nb_pmu;
  57static struct pmu amd_llc_pmu;
  58
  59static cpumask_t amd_nb_active_mask;
  60static cpumask_t amd_llc_active_mask;
  61
  62static bool is_nb_event(struct perf_event *event)
  63{
  64        return event->pmu->type == amd_nb_pmu.type;
  65}
  66
  67static bool is_llc_event(struct perf_event *event)
  68{
  69        return event->pmu->type == amd_llc_pmu.type;
  70}
  71
  72static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
  73{
  74        if (is_nb_event(event) && amd_uncore_nb)
  75                return *per_cpu_ptr(amd_uncore_nb, event->cpu);
  76        else if (is_llc_event(event) && amd_uncore_llc)
  77                return *per_cpu_ptr(amd_uncore_llc, event->cpu);
  78
  79        return NULL;
  80}
  81
  82static void amd_uncore_read(struct perf_event *event)
  83{
  84        struct hw_perf_event *hwc = &event->hw;
  85        u64 prev, new;
  86        s64 delta;
  87
  88        /*
  89         * since we do not enable counter overflow interrupts,
  90         * we do not have to worry about prev_count changing on us
  91         */
  92
  93        prev = local64_read(&hwc->prev_count);
  94        rdpmcl(hwc->event_base_rdpmc, new);
  95        local64_set(&hwc->prev_count, new);
  96        delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
  97        delta >>= COUNTER_SHIFT;
  98        local64_add(delta, &event->count);
  99}
 100
 101static void amd_uncore_start(struct perf_event *event, int flags)
 102{
 103        struct hw_perf_event *hwc = &event->hw;
 104
 105        if (flags & PERF_EF_RELOAD)
 106                wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
 107
 108        hwc->state = 0;
 109        wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
 110        perf_event_update_userpage(event);
 111}
 112
 113static void amd_uncore_stop(struct perf_event *event, int flags)
 114{
 115        struct hw_perf_event *hwc = &event->hw;
 116
 117        wrmsrl(hwc->config_base, hwc->config);
 118        hwc->state |= PERF_HES_STOPPED;
 119
 120        if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
 121                amd_uncore_read(event);
 122                hwc->state |= PERF_HES_UPTODATE;
 123        }
 124}
 125
 126static int amd_uncore_add(struct perf_event *event, int flags)
 127{
 128        int i;
 129        struct amd_uncore *uncore = event_to_amd_uncore(event);
 130        struct hw_perf_event *hwc = &event->hw;
 131
 132        /* are we already assigned? */
 133        if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
 134                goto out;
 135
 136        for (i = 0; i < uncore->num_counters; i++) {
 137                if (uncore->events[i] == event) {
 138                        hwc->idx = i;
 139                        goto out;
 140                }
 141        }
 142
 143        /* if not, take the first available counter */
 144        hwc->idx = -1;
 145        for (i = 0; i < uncore->num_counters; i++) {
 146                if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
 147                        hwc->idx = i;
 148                        break;
 149                }
 150        }
 151
 152out:
 153        if (hwc->idx == -1)
 154                return -EBUSY;
 155
 156        hwc->config_base = uncore->msr_base + (2 * hwc->idx);
 157        hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
 158        hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
 159        hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 160
 161        if (flags & PERF_EF_START)
 162                amd_uncore_start(event, PERF_EF_RELOAD);
 163
 164        return 0;
 165}
 166
 167static void amd_uncore_del(struct perf_event *event, int flags)
 168{
 169        int i;
 170        struct amd_uncore *uncore = event_to_amd_uncore(event);
 171        struct hw_perf_event *hwc = &event->hw;
 172
 173        amd_uncore_stop(event, PERF_EF_UPDATE);
 174
 175        for (i = 0; i < uncore->num_counters; i++) {
 176                if (cmpxchg(&uncore->events[i], event, NULL) == event)
 177                        break;
 178        }
 179
 180        hwc->idx = -1;
 181}
 182
 183/*
 184 * Return a full thread and slice mask unless user
 185 * has provided them
 186 */
 187static u64 l3_thread_slice_mask(u64 config)
 188{
 189        if (boot_cpu_data.x86 <= 0x18)
 190                return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
 191                       ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
 192
 193        /*
 194         * If the user doesn't specify a threadmask, they're not trying to
 195         * count core 0, so we enable all cores & threads.
 196         * We'll also assume that they want to count slice 0 if they specify
 197         * a threadmask and leave sliceid and enallslices unpopulated.
 198         */
 199        if (!(config & AMD64_L3_F19H_THREAD_MASK))
 200                return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
 201                       AMD64_L3_EN_ALL_CORES;
 202
 203        return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
 204                         AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
 205                         AMD64_L3_COREID_MASK);
 206}
 207
 208static int amd_uncore_event_init(struct perf_event *event)
 209{
 210        struct amd_uncore *uncore;
 211        struct hw_perf_event *hwc = &event->hw;
 212
 213        if (event->attr.type != event->pmu->type)
 214                return -ENOENT;
 215
 216        /*
 217         * NB and Last level cache counters (MSRs) are shared across all cores
 218         * that share the same NB / Last level cache.  On family 16h and below,
 219         * Interrupts can be directed to a single target core, however, event
 220         * counts generated by processes running on other cores cannot be masked
 221         * out. So we do not support sampling and per-thread events via
 222         * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
 223         */
 224        hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
 225        hwc->idx = -1;
 226
 227        if (event->cpu < 0)
 228                return -EINVAL;
 229
 230        /*
 231         * SliceMask and ThreadMask need to be set for certain L3 events.
 232         * For other events, the two fields do not affect the count.
 233         */
 234        if (l3_mask && is_llc_event(event))
 235                hwc->config |= l3_thread_slice_mask(event->attr.config);
 236
 237        uncore = event_to_amd_uncore(event);
 238        if (!uncore)
 239                return -ENODEV;
 240
 241        /*
 242         * since request can come in to any of the shared cores, we will remap
 243         * to a single common cpu.
 244         */
 245        event->cpu = uncore->cpu;
 246
 247        return 0;
 248}
 249
 250static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
 251                                            struct device_attribute *attr,
 252                                            char *buf)
 253{
 254        cpumask_t *active_mask;
 255        struct pmu *pmu = dev_get_drvdata(dev);
 256
 257        if (pmu->type == amd_nb_pmu.type)
 258                active_mask = &amd_nb_active_mask;
 259        else if (pmu->type == amd_llc_pmu.type)
 260                active_mask = &amd_llc_active_mask;
 261        else
 262                return 0;
 263
 264        return cpumap_print_to_pagebuf(true, buf, active_mask);
 265}
 266static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
 267
 268static struct attribute *amd_uncore_attrs[] = {
 269        &dev_attr_cpumask.attr,
 270        NULL,
 271};
 272
 273static struct attribute_group amd_uncore_attr_group = {
 274        .attrs = amd_uncore_attrs,
 275};
 276
 277#define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format)                 \
 278static ssize_t __uncore_##_var##_show(struct device *dev,               \
 279                                struct device_attribute *attr,          \
 280                                char *page)                             \
 281{                                                                       \
 282        BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);                     \
 283        return sprintf(page, _format "\n");                             \
 284}                                                                       \
 285static struct device_attribute format_attr_##_var =                     \
 286        __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
 287
 288DEFINE_UNCORE_FORMAT_ATTR(event12,      event,          "config:0-7,32-35");
 289DEFINE_UNCORE_FORMAT_ATTR(event14,      event,          "config:0-7,32-35,59-60"); /* F17h+ DF */
 290DEFINE_UNCORE_FORMAT_ATTR(event8,       event,          "config:0-7");             /* F17h+ L3 */
 291DEFINE_UNCORE_FORMAT_ATTR(umask,        umask,          "config:8-15");
 292DEFINE_UNCORE_FORMAT_ATTR(coreid,       coreid,         "config:42-44");           /* F19h L3 */
 293DEFINE_UNCORE_FORMAT_ATTR(slicemask,    slicemask,      "config:48-51");           /* F17h L3 */
 294DEFINE_UNCORE_FORMAT_ATTR(threadmask8,  threadmask,     "config:56-63");           /* F17h L3 */
 295DEFINE_UNCORE_FORMAT_ATTR(threadmask2,  threadmask,     "config:56-57");           /* F19h L3 */
 296DEFINE_UNCORE_FORMAT_ATTR(enallslices,  enallslices,    "config:46");              /* F19h L3 */
 297DEFINE_UNCORE_FORMAT_ATTR(enallcores,   enallcores,     "config:47");              /* F19h L3 */
 298DEFINE_UNCORE_FORMAT_ATTR(sliceid,      sliceid,        "config:48-50");           /* F19h L3 */
 299
 300static struct attribute *amd_uncore_df_format_attr[] = {
 301        &format_attr_event12.attr, /* event14 if F17h+ */
 302        &format_attr_umask.attr,
 303        NULL,
 304};
 305
 306static struct attribute *amd_uncore_l3_format_attr[] = {
 307        &format_attr_event12.attr, /* event8 if F17h+ */
 308        &format_attr_umask.attr,
 309        NULL, /* slicemask if F17h,     coreid if F19h */
 310        NULL, /* threadmask8 if F17h,   enallslices if F19h */
 311        NULL, /*                        enallcores if F19h */
 312        NULL, /*                        sliceid if F19h */
 313        NULL, /*                        threadmask2 if F19h */
 314        NULL,
 315};
 316
 317static struct attribute_group amd_uncore_df_format_group = {
 318        .name = "format",
 319        .attrs = amd_uncore_df_format_attr,
 320};
 321
 322static struct attribute_group amd_uncore_l3_format_group = {
 323        .name = "format",
 324        .attrs = amd_uncore_l3_format_attr,
 325};
 326
 327static const struct attribute_group *amd_uncore_df_attr_groups[] = {
 328        &amd_uncore_attr_group,
 329        &amd_uncore_df_format_group,
 330        NULL,
 331};
 332
 333static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
 334        &amd_uncore_attr_group,
 335        &amd_uncore_l3_format_group,
 336        NULL,
 337};
 338
 339static struct pmu amd_nb_pmu = {
 340        .task_ctx_nr    = perf_invalid_context,
 341        .attr_groups    = amd_uncore_df_attr_groups,
 342        .name           = "amd_nb",
 343        .event_init     = amd_uncore_event_init,
 344        .add            = amd_uncore_add,
 345        .del            = amd_uncore_del,
 346        .start          = amd_uncore_start,
 347        .stop           = amd_uncore_stop,
 348        .read           = amd_uncore_read,
 349        .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
 350};
 351
 352static struct pmu amd_llc_pmu = {
 353        .task_ctx_nr    = perf_invalid_context,
 354        .attr_groups    = amd_uncore_l3_attr_groups,
 355        .name           = "amd_l2",
 356        .event_init     = amd_uncore_event_init,
 357        .add            = amd_uncore_add,
 358        .del            = amd_uncore_del,
 359        .start          = amd_uncore_start,
 360        .stop           = amd_uncore_stop,
 361        .read           = amd_uncore_read,
 362        .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
 363};
 364
 365static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
 366{
 367        return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
 368                        cpu_to_node(cpu));
 369}
 370
 371static int amd_uncore_cpu_up_prepare(unsigned int cpu)
 372{
 373        struct amd_uncore *uncore_nb = NULL, *uncore_llc;
 374
 375        if (amd_uncore_nb) {
 376                uncore_nb = amd_uncore_alloc(cpu);
 377                if (!uncore_nb)
 378                        goto fail;
 379                uncore_nb->cpu = cpu;
 380                uncore_nb->num_counters = num_counters_nb;
 381                uncore_nb->rdpmc_base = RDPMC_BASE_NB;
 382                uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
 383                uncore_nb->active_mask = &amd_nb_active_mask;
 384                uncore_nb->pmu = &amd_nb_pmu;
 385                uncore_nb->id = -1;
 386                *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
 387        }
 388
 389        if (amd_uncore_llc) {
 390                uncore_llc = amd_uncore_alloc(cpu);
 391                if (!uncore_llc)
 392                        goto fail;
 393                uncore_llc->cpu = cpu;
 394                uncore_llc->num_counters = num_counters_llc;
 395                uncore_llc->rdpmc_base = RDPMC_BASE_LLC;
 396                uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL;
 397                uncore_llc->active_mask = &amd_llc_active_mask;
 398                uncore_llc->pmu = &amd_llc_pmu;
 399                uncore_llc->id = -1;
 400                *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc;
 401        }
 402
 403        return 0;
 404
 405fail:
 406        if (amd_uncore_nb)
 407                *per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
 408        kfree(uncore_nb);
 409        return -ENOMEM;
 410}
 411
 412static struct amd_uncore *
 413amd_uncore_find_online_sibling(struct amd_uncore *this,
 414                               struct amd_uncore * __percpu *uncores)
 415{
 416        unsigned int cpu;
 417        struct amd_uncore *that;
 418
 419        for_each_online_cpu(cpu) {
 420                that = *per_cpu_ptr(uncores, cpu);
 421
 422                if (!that)
 423                        continue;
 424
 425                if (this == that)
 426                        continue;
 427
 428                if (this->id == that->id) {
 429                        hlist_add_head(&this->node, &uncore_unused_list);
 430                        this = that;
 431                        break;
 432                }
 433        }
 434
 435        this->refcnt++;
 436        return this;
 437}
 438
 439static int amd_uncore_cpu_starting(unsigned int cpu)
 440{
 441        unsigned int eax, ebx, ecx, edx;
 442        struct amd_uncore *uncore;
 443
 444        if (amd_uncore_nb) {
 445                uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
 446                cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
 447                uncore->id = ecx & 0xff;
 448
 449                uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
 450                *per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
 451        }
 452
 453        if (amd_uncore_llc) {
 454                uncore = *per_cpu_ptr(amd_uncore_llc, cpu);
 455                uncore->id = per_cpu(cpu_llc_id, cpu);
 456
 457                uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc);
 458                *per_cpu_ptr(amd_uncore_llc, cpu) = uncore;
 459        }
 460
 461        return 0;
 462}
 463
 464static void uncore_clean_online(void)
 465{
 466        struct amd_uncore *uncore;
 467        struct hlist_node *n;
 468
 469        hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) {
 470                hlist_del(&uncore->node);
 471                kfree(uncore);
 472        }
 473}
 474
 475static void uncore_online(unsigned int cpu,
 476                          struct amd_uncore * __percpu *uncores)
 477{
 478        struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
 479
 480        uncore_clean_online();
 481
 482        if (cpu == uncore->cpu)
 483                cpumask_set_cpu(cpu, uncore->active_mask);
 484}
 485
 486static int amd_uncore_cpu_online(unsigned int cpu)
 487{
 488        if (amd_uncore_nb)
 489                uncore_online(cpu, amd_uncore_nb);
 490
 491        if (amd_uncore_llc)
 492                uncore_online(cpu, amd_uncore_llc);
 493
 494        return 0;
 495}
 496
 497static void uncore_down_prepare(unsigned int cpu,
 498                                struct amd_uncore * __percpu *uncores)
 499{
 500        unsigned int i;
 501        struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);
 502
 503        if (this->cpu != cpu)
 504                return;
 505
 506        /* this cpu is going down, migrate to a shared sibling if possible */
 507        for_each_online_cpu(i) {
 508                struct amd_uncore *that = *per_cpu_ptr(uncores, i);
 509
 510                if (cpu == i)
 511                        continue;
 512
 513                if (this == that) {
 514                        perf_pmu_migrate_context(this->pmu, cpu, i);
 515                        cpumask_clear_cpu(cpu, that->active_mask);
 516                        cpumask_set_cpu(i, that->active_mask);
 517                        that->cpu = i;
 518                        break;
 519                }
 520        }
 521}
 522
 523static int amd_uncore_cpu_down_prepare(unsigned int cpu)
 524{
 525        if (amd_uncore_nb)
 526                uncore_down_prepare(cpu, amd_uncore_nb);
 527
 528        if (amd_uncore_llc)
 529                uncore_down_prepare(cpu, amd_uncore_llc);
 530
 531        return 0;
 532}
 533
 534static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
 535{
 536        struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
 537
 538        if (cpu == uncore->cpu)
 539                cpumask_clear_cpu(cpu, uncore->active_mask);
 540
 541        if (!--uncore->refcnt)
 542                kfree(uncore);
 543        *per_cpu_ptr(uncores, cpu) = NULL;
 544}
 545
 546static int amd_uncore_cpu_dead(unsigned int cpu)
 547{
 548        if (amd_uncore_nb)
 549                uncore_dead(cpu, amd_uncore_nb);
 550
 551        if (amd_uncore_llc)
 552                uncore_dead(cpu, amd_uncore_llc);
 553
 554        return 0;
 555}
 556
 557static int __init amd_uncore_init(void)
 558{
 559        struct attribute **df_attr = amd_uncore_df_format_attr;
 560        struct attribute **l3_attr = amd_uncore_l3_format_attr;
 561        int ret = -ENODEV;
 562
 563        if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 564            boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 565                return -ENODEV;
 566
 567        if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
 568                return -ENODEV;
 569
 570        num_counters_nb = NUM_COUNTERS_NB;
 571        num_counters_llc = NUM_COUNTERS_L2;
 572        if (boot_cpu_data.x86 >= 0x17) {
 573                /*
 574                 * For F17h and above, the Northbridge counters are
 575                 * repurposed as Data Fabric counters. Also, L3
 576                 * counters are supported too. The PMUs are exported
 577                 * based on family as either L2 or L3 and NB or DF.
 578                 */
 579                num_counters_llc          = NUM_COUNTERS_L3;
 580                amd_nb_pmu.name           = "amd_df";
 581                amd_llc_pmu.name          = "amd_l3";
 582                l3_mask                   = true;
 583        }
 584
 585        if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
 586                if (boot_cpu_data.x86 >= 0x17)
 587                        *df_attr = &format_attr_event14.attr;
 588
 589                amd_uncore_nb = alloc_percpu(struct amd_uncore *);
 590                if (!amd_uncore_nb) {
 591                        ret = -ENOMEM;
 592                        goto fail_nb;
 593                }
 594                ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
 595                if (ret)
 596                        goto fail_nb;
 597
 598                pr_info("%d %s %s counters detected\n", num_counters_nb,
 599                        boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ?  "HYGON" : "",
 600                        amd_nb_pmu.name);
 601
 602                ret = 0;
 603        }
 604
 605        if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) {
 606                if (boot_cpu_data.x86 >= 0x19) {
 607                        *l3_attr++ = &format_attr_event8.attr;
 608                        *l3_attr++ = &format_attr_umask.attr;
 609                        *l3_attr++ = &format_attr_coreid.attr;
 610                        *l3_attr++ = &format_attr_enallslices.attr;
 611                        *l3_attr++ = &format_attr_enallcores.attr;
 612                        *l3_attr++ = &format_attr_sliceid.attr;
 613                        *l3_attr++ = &format_attr_threadmask2.attr;
 614                } else if (boot_cpu_data.x86 >= 0x17) {
 615                        *l3_attr++ = &format_attr_event8.attr;
 616                        *l3_attr++ = &format_attr_umask.attr;
 617                        *l3_attr++ = &format_attr_slicemask.attr;
 618                        *l3_attr++ = &format_attr_threadmask8.attr;
 619                }
 620
 621                amd_uncore_llc = alloc_percpu(struct amd_uncore *);
 622                if (!amd_uncore_llc) {
 623                        ret = -ENOMEM;
 624                        goto fail_llc;
 625                }
 626                ret = perf_pmu_register(&amd_llc_pmu, amd_llc_pmu.name, -1);
 627                if (ret)
 628                        goto fail_llc;
 629
 630                pr_info("%d %s %s counters detected\n", num_counters_llc,
 631                        boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ?  "HYGON" : "",
 632                        amd_llc_pmu.name);
 633                ret = 0;
 634        }
 635
 636        /*
 637         * Install callbacks. Core will call them for each online cpu.
 638         */
 639        if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
 640                              "perf/x86/amd/uncore:prepare",
 641                              amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
 642                goto fail_llc;
 643
 644        if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
 645                              "perf/x86/amd/uncore:starting",
 646                              amd_uncore_cpu_starting, NULL))
 647                goto fail_prep;
 648        if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
 649                              "perf/x86/amd/uncore:online",
 650                              amd_uncore_cpu_online,
 651                              amd_uncore_cpu_down_prepare))
 652                goto fail_start;
 653        return 0;
 654
 655fail_start:
 656        cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
 657fail_prep:
 658        cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
 659fail_llc:
 660        if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
 661                perf_pmu_unregister(&amd_nb_pmu);
 662        if (amd_uncore_llc)
 663                free_percpu(amd_uncore_llc);
 664fail_nb:
 665        if (amd_uncore_nb)
 666                free_percpu(amd_uncore_nb);
 667
 668        return ret;
 669}
 670device_initcall(amd_uncore_init);
 671