linux/drivers/pci/controller/pci-xgene-msi.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * APM X-Gene MSI Driver
   4 *
   5 * Copyright (c) 2014, Applied Micro Circuits Corporation
   6 * Author: Tanmay Inamdar <tinamdar@apm.com>
   7 *         Duc Dang <dhdang@apm.com>
   8 */
   9#include <linux/cpu.h>
  10#include <linux/interrupt.h>
  11#include <linux/module.h>
  12#include <linux/msi.h>
  13#include <linux/of_irq.h>
  14#include <linux/irqchip/chained_irq.h>
  15#include <linux/pci.h>
  16#include <linux/platform_device.h>
  17#include <linux/of_pci.h>
  18
  19#define MSI_IR0                 0x000000
  20#define MSI_INT0                0x800000
  21#define IDX_PER_GROUP           8
  22#define IRQS_PER_IDX            16
  23#define NR_HW_IRQS              16
  24#define NR_MSI_VEC              (IDX_PER_GROUP * IRQS_PER_IDX * NR_HW_IRQS)
  25
  26struct xgene_msi_group {
  27        struct xgene_msi        *msi;
  28        int                     gic_irq;
  29        u32                     msi_grp;
  30};
  31
  32struct xgene_msi {
  33        struct device_node      *node;
  34        struct irq_domain       *inner_domain;
  35        struct irq_domain       *msi_domain;
  36        u64                     msi_addr;
  37        void __iomem            *msi_regs;
  38        unsigned long           *bitmap;
  39        struct mutex            bitmap_lock;
  40        struct xgene_msi_group  *msi_groups;
  41        int                     num_cpus;
  42};
  43
  44/* Global data */
  45static struct xgene_msi xgene_msi_ctrl;
  46
  47static struct irq_chip xgene_msi_top_irq_chip = {
  48        .name           = "X-Gene1 MSI",
  49        .irq_enable     = pci_msi_unmask_irq,
  50        .irq_disable    = pci_msi_mask_irq,
  51        .irq_mask       = pci_msi_mask_irq,
  52        .irq_unmask     = pci_msi_unmask_irq,
  53};
  54
  55static struct  msi_domain_info xgene_msi_domain_info = {
  56        .flags  = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
  57                  MSI_FLAG_PCI_MSIX),
  58        .chip   = &xgene_msi_top_irq_chip,
  59};
  60
  61/*
  62 * X-Gene v1 has 16 groups of MSI termination registers MSInIRx, where
  63 * n is group number (0..F), x is index of registers in each group (0..7)
  64 * The register layout is as follows:
  65 * MSI0IR0                      base_addr
  66 * MSI0IR1                      base_addr +  0x10000
  67 * ...                          ...
  68 * MSI0IR6                      base_addr +  0x60000
  69 * MSI0IR7                      base_addr +  0x70000
  70 * MSI1IR0                      base_addr +  0x80000
  71 * MSI1IR1                      base_addr +  0x90000
  72 * ...                          ...
  73 * MSI1IR7                      base_addr +  0xF0000
  74 * MSI2IR0                      base_addr + 0x100000
  75 * ...                          ...
  76 * MSIFIR0                      base_addr + 0x780000
  77 * MSIFIR1                      base_addr + 0x790000
  78 * ...                          ...
  79 * MSIFIR7                      base_addr + 0x7F0000
  80 * MSIINT0                      base_addr + 0x800000
  81 * MSIINT1                      base_addr + 0x810000
  82 * ...                          ...
  83 * MSIINTF                      base_addr + 0x8F0000
  84 *
  85 * Each index register supports 16 MSI vectors (0..15) to generate interrupt.
  86 * There are total 16 GIC IRQs assigned for these 16 groups of MSI termination
  87 * registers.
  88 *
  89 * Each MSI termination group has 1 MSIINTn register (n is 0..15) to indicate
  90 * the MSI pending status caused by 1 of its 8 index registers.
  91 */
  92
  93/* MSInIRx read helper */
  94static u32 xgene_msi_ir_read(struct xgene_msi *msi,
  95                                    u32 msi_grp, u32 msir_idx)
  96{
  97        return readl_relaxed(msi->msi_regs + MSI_IR0 +
  98                              (msi_grp << 19) + (msir_idx << 16));
  99}
 100
 101/* MSIINTn read helper */
 102static u32 xgene_msi_int_read(struct xgene_msi *msi, u32 msi_grp)
 103{
 104        return readl_relaxed(msi->msi_regs + MSI_INT0 + (msi_grp << 16));
 105}
 106
 107/*
 108 * With 2048 MSI vectors supported, the MSI message can be constructed using
 109 * following scheme:
 110 * - Divide into 8 256-vector groups
 111 *              Group 0: 0-255
 112 *              Group 1: 256-511
 113 *              Group 2: 512-767
 114 *              ...
 115 *              Group 7: 1792-2047
 116 * - Each 256-vector group is divided into 16 16-vector groups
 117 *      As an example: 16 16-vector groups for 256-vector group 0-255 is
 118 *              Group 0: 0-15
 119 *              Group 1: 16-32
 120 *              ...
 121 *              Group 15: 240-255
 122 * - The termination address of MSI vector in 256-vector group n and 16-vector
 123 *   group x is the address of MSIxIRn
 124 * - The data for MSI vector in 16-vector group x is x
 125 */
 126static u32 hwirq_to_reg_set(unsigned long hwirq)
 127{
 128        return (hwirq / (NR_HW_IRQS * IRQS_PER_IDX));
 129}
 130
 131static u32 hwirq_to_group(unsigned long hwirq)
 132{
 133        return (hwirq % NR_HW_IRQS);
 134}
 135
 136static u32 hwirq_to_msi_data(unsigned long hwirq)
 137{
 138        return ((hwirq / NR_HW_IRQS) % IRQS_PER_IDX);
 139}
 140
 141static void xgene_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 142{
 143        struct xgene_msi *msi = irq_data_get_irq_chip_data(data);
 144        u32 reg_set = hwirq_to_reg_set(data->hwirq);
 145        u32 group = hwirq_to_group(data->hwirq);
 146        u64 target_addr = msi->msi_addr + (((8 * group) + reg_set) << 16);
 147
 148        msg->address_hi = upper_32_bits(target_addr);
 149        msg->address_lo = lower_32_bits(target_addr);
 150        msg->data = hwirq_to_msi_data(data->hwirq);
 151}
 152
 153/*
 154 * X-Gene v1 only has 16 MSI GIC IRQs for 2048 MSI vectors.  To maintain
 155 * the expected behaviour of .set_affinity for each MSI interrupt, the 16
 156 * MSI GIC IRQs are statically allocated to 8 X-Gene v1 cores (2 GIC IRQs
 157 * for each core).  The MSI vector is moved fom 1 MSI GIC IRQ to another
 158 * MSI GIC IRQ to steer its MSI interrupt to correct X-Gene v1 core.  As a
 159 * consequence, the total MSI vectors that X-Gene v1 supports will be
 160 * reduced to 256 (2048/8) vectors.
 161 */
 162static int hwirq_to_cpu(unsigned long hwirq)
 163{
 164        return (hwirq % xgene_msi_ctrl.num_cpus);
 165}
 166
 167static unsigned long hwirq_to_canonical_hwirq(unsigned long hwirq)
 168{
 169        return (hwirq - hwirq_to_cpu(hwirq));
 170}
 171
 172static int xgene_msi_set_affinity(struct irq_data *irqdata,
 173                                  const struct cpumask *mask, bool force)
 174{
 175        int target_cpu = cpumask_first(mask);
 176        int curr_cpu;
 177
 178        curr_cpu = hwirq_to_cpu(irqdata->hwirq);
 179        if (curr_cpu == target_cpu)
 180                return IRQ_SET_MASK_OK_DONE;
 181
 182        /* Update MSI number to target the new CPU */
 183        irqdata->hwirq = hwirq_to_canonical_hwirq(irqdata->hwirq) + target_cpu;
 184
 185        return IRQ_SET_MASK_OK;
 186}
 187
 188static struct irq_chip xgene_msi_bottom_irq_chip = {
 189        .name                   = "MSI",
 190        .irq_set_affinity       = xgene_msi_set_affinity,
 191        .irq_compose_msi_msg    = xgene_compose_msi_msg,
 192};
 193
 194static int xgene_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 195                                  unsigned int nr_irqs, void *args)
 196{
 197        struct xgene_msi *msi = domain->host_data;
 198        int msi_irq;
 199
 200        mutex_lock(&msi->bitmap_lock);
 201
 202        msi_irq = bitmap_find_next_zero_area(msi->bitmap, NR_MSI_VEC, 0,
 203                                             msi->num_cpus, 0);
 204        if (msi_irq < NR_MSI_VEC)
 205                bitmap_set(msi->bitmap, msi_irq, msi->num_cpus);
 206        else
 207                msi_irq = -ENOSPC;
 208
 209        mutex_unlock(&msi->bitmap_lock);
 210
 211        if (msi_irq < 0)
 212                return msi_irq;
 213
 214        irq_domain_set_info(domain, virq, msi_irq,
 215                            &xgene_msi_bottom_irq_chip, domain->host_data,
 216                            handle_simple_irq, NULL, NULL);
 217
 218        return 0;
 219}
 220
 221static void xgene_irq_domain_free(struct irq_domain *domain,
 222                                  unsigned int virq, unsigned int nr_irqs)
 223{
 224        struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 225        struct xgene_msi *msi = irq_data_get_irq_chip_data(d);
 226        u32 hwirq;
 227
 228        mutex_lock(&msi->bitmap_lock);
 229
 230        hwirq = hwirq_to_canonical_hwirq(d->hwirq);
 231        bitmap_clear(msi->bitmap, hwirq, msi->num_cpus);
 232
 233        mutex_unlock(&msi->bitmap_lock);
 234
 235        irq_domain_free_irqs_parent(domain, virq, nr_irqs);
 236}
 237
 238static const struct irq_domain_ops msi_domain_ops = {
 239        .alloc  = xgene_irq_domain_alloc,
 240        .free   = xgene_irq_domain_free,
 241};
 242
 243static int xgene_allocate_domains(struct xgene_msi *msi)
 244{
 245        msi->inner_domain = irq_domain_add_linear(NULL, NR_MSI_VEC,
 246                                                  &msi_domain_ops, msi);
 247        if (!msi->inner_domain)
 248                return -ENOMEM;
 249
 250        msi->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(msi->node),
 251                                                    &xgene_msi_domain_info,
 252                                                    msi->inner_domain);
 253
 254        if (!msi->msi_domain) {
 255                irq_domain_remove(msi->inner_domain);
 256                return -ENOMEM;
 257        }
 258
 259        return 0;
 260}
 261
 262static void xgene_free_domains(struct xgene_msi *msi)
 263{
 264        if (msi->msi_domain)
 265                irq_domain_remove(msi->msi_domain);
 266        if (msi->inner_domain)
 267                irq_domain_remove(msi->inner_domain);
 268}
 269
 270static int xgene_msi_init_allocator(struct xgene_msi *xgene_msi)
 271{
 272        int size = BITS_TO_LONGS(NR_MSI_VEC) * sizeof(long);
 273
 274        xgene_msi->bitmap = kzalloc(size, GFP_KERNEL);
 275        if (!xgene_msi->bitmap)
 276                return -ENOMEM;
 277
 278        mutex_init(&xgene_msi->bitmap_lock);
 279
 280        xgene_msi->msi_groups = kcalloc(NR_HW_IRQS,
 281                                        sizeof(struct xgene_msi_group),
 282                                        GFP_KERNEL);
 283        if (!xgene_msi->msi_groups)
 284                return -ENOMEM;
 285
 286        return 0;
 287}
 288
 289static void xgene_msi_isr(struct irq_desc *desc)
 290{
 291        struct irq_chip *chip = irq_desc_get_chip(desc);
 292        struct xgene_msi_group *msi_groups;
 293        struct xgene_msi *xgene_msi;
 294        unsigned int virq;
 295        int msir_index, msir_val, hw_irq;
 296        u32 intr_index, grp_select, msi_grp;
 297
 298        chained_irq_enter(chip, desc);
 299
 300        msi_groups = irq_desc_get_handler_data(desc);
 301        xgene_msi = msi_groups->msi;
 302        msi_grp = msi_groups->msi_grp;
 303
 304        /*
 305         * MSIINTn (n is 0..F) indicates if there is a pending MSI interrupt
 306         * If bit x of this register is set (x is 0..7), one or more interupts
 307         * corresponding to MSInIRx is set.
 308         */
 309        grp_select = xgene_msi_int_read(xgene_msi, msi_grp);
 310        while (grp_select) {
 311                msir_index = ffs(grp_select) - 1;
 312                /*
 313                 * Calculate MSInIRx address to read to check for interrupts
 314                 * (refer to termination address and data assignment
 315                 * described in xgene_compose_msi_msg() )
 316                 */
 317                msir_val = xgene_msi_ir_read(xgene_msi, msi_grp, msir_index);
 318                while (msir_val) {
 319                        intr_index = ffs(msir_val) - 1;
 320                        /*
 321                         * Calculate MSI vector number (refer to the termination
 322                         * address and data assignment described in
 323                         * xgene_compose_msi_msg function)
 324                         */
 325                        hw_irq = (((msir_index * IRQS_PER_IDX) + intr_index) *
 326                                 NR_HW_IRQS) + msi_grp;
 327                        /*
 328                         * As we have multiple hw_irq that maps to single MSI,
 329                         * always look up the virq using the hw_irq as seen from
 330                         * CPU0
 331                         */
 332                        hw_irq = hwirq_to_canonical_hwirq(hw_irq);
 333                        virq = irq_find_mapping(xgene_msi->inner_domain, hw_irq);
 334                        WARN_ON(!virq);
 335                        if (virq != 0)
 336                                generic_handle_irq(virq);
 337                        msir_val &= ~(1 << intr_index);
 338                }
 339                grp_select &= ~(1 << msir_index);
 340
 341                if (!grp_select) {
 342                        /*
 343                         * We handled all interrupts happened in this group,
 344                         * resample this group MSI_INTx register in case
 345                         * something else has been made pending in the meantime
 346                         */
 347                        grp_select = xgene_msi_int_read(xgene_msi, msi_grp);
 348                }
 349        }
 350
 351        chained_irq_exit(chip, desc);
 352}
 353
 354static enum cpuhp_state pci_xgene_online;
 355
 356static int xgene_msi_remove(struct platform_device *pdev)
 357{
 358        struct xgene_msi *msi = platform_get_drvdata(pdev);
 359
 360        if (pci_xgene_online)
 361                cpuhp_remove_state(pci_xgene_online);
 362        cpuhp_remove_state(CPUHP_PCI_XGENE_DEAD);
 363
 364        kfree(msi->msi_groups);
 365
 366        kfree(msi->bitmap);
 367        msi->bitmap = NULL;
 368
 369        xgene_free_domains(msi);
 370
 371        return 0;
 372}
 373
 374static int xgene_msi_hwirq_alloc(unsigned int cpu)
 375{
 376        struct xgene_msi *msi = &xgene_msi_ctrl;
 377        struct xgene_msi_group *msi_group;
 378        cpumask_var_t mask;
 379        int i;
 380        int err;
 381
 382        for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) {
 383                msi_group = &msi->msi_groups[i];
 384                if (!msi_group->gic_irq)
 385                        continue;
 386
 387                irq_set_chained_handler_and_data(msi_group->gic_irq,
 388                        xgene_msi_isr, msi_group);
 389
 390                /*
 391                 * Statically allocate MSI GIC IRQs to each CPU core.
 392                 * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated
 393                 * to each core.
 394                 */
 395                if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
 396                        cpumask_clear(mask);
 397                        cpumask_set_cpu(cpu, mask);
 398                        err = irq_set_affinity(msi_group->gic_irq, mask);
 399                        if (err)
 400                                pr_err("failed to set affinity for GIC IRQ");
 401                        free_cpumask_var(mask);
 402                } else {
 403                        pr_err("failed to alloc CPU mask for affinity\n");
 404                        err = -EINVAL;
 405                }
 406
 407                if (err) {
 408                        irq_set_chained_handler_and_data(msi_group->gic_irq,
 409                                                         NULL, NULL);
 410                        return err;
 411                }
 412        }
 413
 414        return 0;
 415}
 416
 417static int xgene_msi_hwirq_free(unsigned int cpu)
 418{
 419        struct xgene_msi *msi = &xgene_msi_ctrl;
 420        struct xgene_msi_group *msi_group;
 421        int i;
 422
 423        for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) {
 424                msi_group = &msi->msi_groups[i];
 425                if (!msi_group->gic_irq)
 426                        continue;
 427
 428                irq_set_chained_handler_and_data(msi_group->gic_irq, NULL,
 429                                                 NULL);
 430        }
 431        return 0;
 432}
 433
 434static const struct of_device_id xgene_msi_match_table[] = {
 435        {.compatible = "apm,xgene1-msi"},
 436        {},
 437};
 438
 439static int xgene_msi_probe(struct platform_device *pdev)
 440{
 441        struct resource *res;
 442        int rc, irq_index;
 443        struct xgene_msi *xgene_msi;
 444        int virt_msir;
 445        u32 msi_val, msi_idx;
 446
 447        xgene_msi = &xgene_msi_ctrl;
 448
 449        platform_set_drvdata(pdev, xgene_msi);
 450
 451        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 452        xgene_msi->msi_regs = devm_ioremap_resource(&pdev->dev, res);
 453        if (IS_ERR(xgene_msi->msi_regs)) {
 454                dev_err(&pdev->dev, "no reg space\n");
 455                rc = PTR_ERR(xgene_msi->msi_regs);
 456                goto error;
 457        }
 458        xgene_msi->msi_addr = res->start;
 459        xgene_msi->node = pdev->dev.of_node;
 460        xgene_msi->num_cpus = num_possible_cpus();
 461
 462        rc = xgene_msi_init_allocator(xgene_msi);
 463        if (rc) {
 464                dev_err(&pdev->dev, "Error allocating MSI bitmap\n");
 465                goto error;
 466        }
 467
 468        rc = xgene_allocate_domains(xgene_msi);
 469        if (rc) {
 470                dev_err(&pdev->dev, "Failed to allocate MSI domain\n");
 471                goto error;
 472        }
 473
 474        for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) {
 475                virt_msir = platform_get_irq(pdev, irq_index);
 476                if (virt_msir < 0) {
 477                        rc = virt_msir;
 478                        goto error;
 479                }
 480                xgene_msi->msi_groups[irq_index].gic_irq = virt_msir;
 481                xgene_msi->msi_groups[irq_index].msi_grp = irq_index;
 482                xgene_msi->msi_groups[irq_index].msi = xgene_msi;
 483        }
 484
 485        /*
 486         * MSInIRx registers are read-to-clear; before registering
 487         * interrupt handlers, read all of them to clear spurious
 488         * interrupts that may occur before the driver is probed.
 489         */
 490        for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) {
 491                for (msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++)
 492                        xgene_msi_ir_read(xgene_msi, irq_index, msi_idx);
 493
 494                /* Read MSIINTn to confirm */
 495                msi_val = xgene_msi_int_read(xgene_msi, irq_index);
 496                if (msi_val) {
 497                        dev_err(&pdev->dev, "Failed to clear spurious IRQ\n");
 498                        rc = -EINVAL;
 499                        goto error;
 500                }
 501        }
 502
 503        rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/xgene:online",
 504                               xgene_msi_hwirq_alloc, NULL);
 505        if (rc < 0)
 506                goto err_cpuhp;
 507        pci_xgene_online = rc;
 508        rc = cpuhp_setup_state(CPUHP_PCI_XGENE_DEAD, "pci/xgene:dead", NULL,
 509                               xgene_msi_hwirq_free);
 510        if (rc)
 511                goto err_cpuhp;
 512
 513        dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n");
 514
 515        return 0;
 516
 517err_cpuhp:
 518        dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
 519error:
 520        xgene_msi_remove(pdev);
 521        return rc;
 522}
 523
 524static struct platform_driver xgene_msi_driver = {
 525        .driver = {
 526                .name = "xgene-msi",
 527                .of_match_table = xgene_msi_match_table,
 528        },
 529        .probe = xgene_msi_probe,
 530        .remove = xgene_msi_remove,
 531};
 532
 533static int __init xgene_pcie_msi_init(void)
 534{
 535        return platform_driver_register(&xgene_msi_driver);
 536}
 537subsys_initcall(xgene_pcie_msi_init);
 538