linux/drivers/pci/probe.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * PCI detection and setup code
   4 */
   5
   6#include <linux/kernel.h>
   7#include <linux/delay.h>
   8#include <linux/init.h>
   9#include <linux/pci.h>
  10#include <linux/msi.h>
  11#include <linux/of_device.h>
  12#include <linux/of_pci.h>
  13#include <linux/pci_hotplug.h>
  14#include <linux/slab.h>
  15#include <linux/module.h>
  16#include <linux/cpumask.h>
  17#include <linux/aer.h>
  18#include <linux/acpi.h>
  19#include <linux/hypervisor.h>
  20#include <linux/irqdomain.h>
  21#include <linux/pm_runtime.h>
  22#include "pci.h"
  23
  24#define CARDBUS_LATENCY_TIMER   176     /* secondary latency timer */
  25#define CARDBUS_RESERVE_BUSNR   3
  26
  27static struct resource busn_resource = {
  28        .name   = "PCI busn",
  29        .start  = 0,
  30        .end    = 255,
  31        .flags  = IORESOURCE_BUS,
  32};
  33
  34/* Ugh.  Need to stop exporting this to modules. */
  35LIST_HEAD(pci_root_buses);
  36EXPORT_SYMBOL(pci_root_buses);
  37
  38static LIST_HEAD(pci_domain_busn_res_list);
  39
  40struct pci_domain_busn_res {
  41        struct list_head list;
  42        struct resource res;
  43        int domain_nr;
  44};
  45
  46static struct resource *get_pci_domain_busn_res(int domain_nr)
  47{
  48        struct pci_domain_busn_res *r;
  49
  50        list_for_each_entry(r, &pci_domain_busn_res_list, list)
  51                if (r->domain_nr == domain_nr)
  52                        return &r->res;
  53
  54        r = kzalloc(sizeof(*r), GFP_KERNEL);
  55        if (!r)
  56                return NULL;
  57
  58        r->domain_nr = domain_nr;
  59        r->res.start = 0;
  60        r->res.end = 0xff;
  61        r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
  62
  63        list_add_tail(&r->list, &pci_domain_busn_res_list);
  64
  65        return &r->res;
  66}
  67
  68/*
  69 * Some device drivers need know if PCI is initiated.
  70 * Basically, we think PCI is not initiated when there
  71 * is no device to be found on the pci_bus_type.
  72 */
  73int no_pci_devices(void)
  74{
  75        struct device *dev;
  76        int no_devices;
  77
  78        dev = bus_find_next_device(&pci_bus_type, NULL);
  79        no_devices = (dev == NULL);
  80        put_device(dev);
  81        return no_devices;
  82}
  83EXPORT_SYMBOL(no_pci_devices);
  84
  85/*
  86 * PCI Bus Class
  87 */
  88static void release_pcibus_dev(struct device *dev)
  89{
  90        struct pci_bus *pci_bus = to_pci_bus(dev);
  91
  92        put_device(pci_bus->bridge);
  93        pci_bus_remove_resources(pci_bus);
  94        pci_release_bus_of_node(pci_bus);
  95        kfree(pci_bus);
  96}
  97
  98static struct class pcibus_class = {
  99        .name           = "pci_bus",
 100        .dev_release    = &release_pcibus_dev,
 101        .dev_groups     = pcibus_groups,
 102};
 103
 104static int __init pcibus_class_init(void)
 105{
 106        return class_register(&pcibus_class);
 107}
 108postcore_initcall(pcibus_class_init);
 109
 110static u64 pci_size(u64 base, u64 maxbase, u64 mask)
 111{
 112        u64 size = mask & maxbase;      /* Find the significant bits */
 113        if (!size)
 114                return 0;
 115
 116        /*
 117         * Get the lowest of them to find the decode size, and from that
 118         * the extent.
 119         */
 120        size = size & ~(size-1);
 121
 122        /*
 123         * base == maxbase can be valid only if the BAR has already been
 124         * programmed with all 1s.
 125         */
 126        if (base == maxbase && ((base | (size - 1)) & mask) != mask)
 127                return 0;
 128
 129        return size;
 130}
 131
 132static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
 133{
 134        u32 mem_type;
 135        unsigned long flags;
 136
 137        if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
 138                flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
 139                flags |= IORESOURCE_IO;
 140                return flags;
 141        }
 142
 143        flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
 144        flags |= IORESOURCE_MEM;
 145        if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
 146                flags |= IORESOURCE_PREFETCH;
 147
 148        mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 149        switch (mem_type) {
 150        case PCI_BASE_ADDRESS_MEM_TYPE_32:
 151                break;
 152        case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 153                /* 1M mem BAR treated as 32-bit BAR */
 154                break;
 155        case PCI_BASE_ADDRESS_MEM_TYPE_64:
 156                flags |= IORESOURCE_MEM_64;
 157                break;
 158        default:
 159                /* mem unknown type treated as 32-bit BAR */
 160                break;
 161        }
 162        return flags;
 163}
 164
 165#define PCI_COMMAND_DECODE_ENABLE       (PCI_COMMAND_MEMORY | PCI_COMMAND_IO)
 166
 167/**
 168 * __pci_read_base - Read a PCI BAR
 169 * @dev: the PCI device
 170 * @type: type of the BAR
 171 * @res: resource buffer to be filled in
 172 * @pos: BAR position in the config space
 173 *
 174 * Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
 175 */
 176int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
 177                    struct resource *res, unsigned int pos)
 178{
 179        u32 l = 0, sz = 0, mask;
 180        u64 l64, sz64, mask64;
 181        u16 orig_cmd;
 182        struct pci_bus_region region, inverted_region;
 183
 184        mask = type ? PCI_ROM_ADDRESS_MASK : ~0;
 185
 186        /* No printks while decoding is disabled! */
 187        if (!dev->mmio_always_on) {
 188                pci_read_config_word(dev, PCI_COMMAND, &orig_cmd);
 189                if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
 190                        pci_write_config_word(dev, PCI_COMMAND,
 191                                orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
 192                }
 193        }
 194
 195        res->name = pci_name(dev);
 196
 197        pci_read_config_dword(dev, pos, &l);
 198        pci_write_config_dword(dev, pos, l | mask);
 199        pci_read_config_dword(dev, pos, &sz);
 200        pci_write_config_dword(dev, pos, l);
 201
 202        /*
 203         * All bits set in sz means the device isn't working properly.
 204         * If the BAR isn't implemented, all bits must be 0.  If it's a
 205         * memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
 206         * 1 must be clear.
 207         */
 208        if (sz == 0xffffffff)
 209                sz = 0;
 210
 211        /*
 212         * I don't know how l can have all bits set.  Copied from old code.
 213         * Maybe it fixes a bug on some ancient platform.
 214         */
 215        if (l == 0xffffffff)
 216                l = 0;
 217
 218        if (type == pci_bar_unknown) {
 219                res->flags = decode_bar(dev, l);
 220                res->flags |= IORESOURCE_SIZEALIGN;
 221                if (res->flags & IORESOURCE_IO) {
 222                        l64 = l & PCI_BASE_ADDRESS_IO_MASK;
 223                        sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
 224                        mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
 225                } else {
 226                        l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
 227                        sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
 228                        mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
 229                }
 230        } else {
 231                if (l & PCI_ROM_ADDRESS_ENABLE)
 232                        res->flags |= IORESOURCE_ROM_ENABLE;
 233                l64 = l & PCI_ROM_ADDRESS_MASK;
 234                sz64 = sz & PCI_ROM_ADDRESS_MASK;
 235                mask64 = PCI_ROM_ADDRESS_MASK;
 236        }
 237
 238        if (res->flags & IORESOURCE_MEM_64) {
 239                pci_read_config_dword(dev, pos + 4, &l);
 240                pci_write_config_dword(dev, pos + 4, ~0);
 241                pci_read_config_dword(dev, pos + 4, &sz);
 242                pci_write_config_dword(dev, pos + 4, l);
 243
 244                l64 |= ((u64)l << 32);
 245                sz64 |= ((u64)sz << 32);
 246                mask64 |= ((u64)~0 << 32);
 247        }
 248
 249        if (!dev->mmio_always_on && (orig_cmd & PCI_COMMAND_DECODE_ENABLE))
 250                pci_write_config_word(dev, PCI_COMMAND, orig_cmd);
 251
 252        if (!sz64)
 253                goto fail;
 254
 255        sz64 = pci_size(l64, sz64, mask64);
 256        if (!sz64) {
 257                pci_info(dev, FW_BUG "reg 0x%x: invalid BAR (can't size)\n",
 258                         pos);
 259                goto fail;
 260        }
 261
 262        if (res->flags & IORESOURCE_MEM_64) {
 263                if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
 264                    && sz64 > 0x100000000ULL) {
 265                        res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
 266                        res->start = 0;
 267                        res->end = 0;
 268                        pci_err(dev, "reg 0x%x: can't handle BAR larger than 4GB (size %#010llx)\n",
 269                                pos, (unsigned long long)sz64);
 270                        goto out;
 271                }
 272
 273                if ((sizeof(pci_bus_addr_t) < 8) && l) {
 274                        /* Above 32-bit boundary; try to reallocate */
 275                        res->flags |= IORESOURCE_UNSET;
 276                        res->start = 0;
 277                        res->end = sz64 - 1;
 278                        pci_info(dev, "reg 0x%x: can't handle BAR above 4GB (bus address %#010llx)\n",
 279                                 pos, (unsigned long long)l64);
 280                        goto out;
 281                }
 282        }
 283
 284        region.start = l64;
 285        region.end = l64 + sz64 - 1;
 286
 287        pcibios_bus_to_resource(dev->bus, res, &region);
 288        pcibios_resource_to_bus(dev->bus, &inverted_region, res);
 289
 290        /*
 291         * If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
 292         * the corresponding resource address (the physical address used by
 293         * the CPU.  Converting that resource address back to a bus address
 294         * should yield the original BAR value:
 295         *
 296         *     resource_to_bus(bus_to_resource(A)) == A
 297         *
 298         * If it doesn't, CPU accesses to "bus_to_resource(A)" will not
 299         * be claimed by the device.
 300         */
 301        if (inverted_region.start != region.start) {
 302                res->flags |= IORESOURCE_UNSET;
 303                res->start = 0;
 304                res->end = region.end - region.start;
 305                pci_info(dev, "reg 0x%x: initial BAR value %#010llx invalid\n",
 306                         pos, (unsigned long long)region.start);
 307        }
 308
 309        goto out;
 310
 311
 312fail:
 313        res->flags = 0;
 314out:
 315        if (res->flags)
 316                pci_info(dev, "reg 0x%x: %pR\n", pos, res);
 317
 318        return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
 319}
 320
 321static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom)
 322{
 323        unsigned int pos, reg;
 324
 325        if (dev->non_compliant_bars)
 326                return;
 327
 328        /* Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero */
 329        if (dev->is_virtfn)
 330                return;
 331
 332        for (pos = 0; pos < howmany; pos++) {
 333                struct resource *res = &dev->resource[pos];
 334                reg = PCI_BASE_ADDRESS_0 + (pos << 2);
 335                pos += __pci_read_base(dev, pci_bar_unknown, res, reg);
 336        }
 337
 338        if (rom) {
 339                struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
 340                dev->rom_base_reg = rom;
 341                res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
 342                                IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
 343                __pci_read_base(dev, pci_bar_mem32, res, rom);
 344        }
 345}
 346
 347static void pci_read_bridge_windows(struct pci_dev *bridge)
 348{
 349        u16 io;
 350        u32 pmem, tmp;
 351
 352        pci_read_config_word(bridge, PCI_IO_BASE, &io);
 353        if (!io) {
 354                pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0);
 355                pci_read_config_word(bridge, PCI_IO_BASE, &io);
 356                pci_write_config_word(bridge, PCI_IO_BASE, 0x0);
 357        }
 358        if (io)
 359                bridge->io_window = 1;
 360
 361        /*
 362         * DECchip 21050 pass 2 errata: the bridge may miss an address
 363         * disconnect boundary by one PCI data phase.  Workaround: do not
 364         * use prefetching on this device.
 365         */
 366        if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001)
 367                return;
 368
 369        pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
 370        if (!pmem) {
 371                pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE,
 372                                               0xffe0fff0);
 373                pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
 374                pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, 0x0);
 375        }
 376        if (!pmem)
 377                return;
 378
 379        bridge->pref_window = 1;
 380
 381        if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
 382
 383                /*
 384                 * Bridge claims to have a 64-bit prefetchable memory
 385                 * window; verify that the upper bits are actually
 386                 * writable.
 387                 */
 388                pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &pmem);
 389                pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32,
 390                                       0xffffffff);
 391                pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp);
 392                pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, pmem);
 393                if (tmp)
 394                        bridge->pref_64_window = 1;
 395        }
 396}
 397
 398static void pci_read_bridge_io(struct pci_bus *child)
 399{
 400        struct pci_dev *dev = child->self;
 401        u8 io_base_lo, io_limit_lo;
 402        unsigned long io_mask, io_granularity, base, limit;
 403        struct pci_bus_region region;
 404        struct resource *res;
 405
 406        io_mask = PCI_IO_RANGE_MASK;
 407        io_granularity = 0x1000;
 408        if (dev->io_window_1k) {
 409                /* Support 1K I/O space granularity */
 410                io_mask = PCI_IO_1K_RANGE_MASK;
 411                io_granularity = 0x400;
 412        }
 413
 414        res = child->resource[0];
 415        pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo);
 416        pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo);
 417        base = (io_base_lo & io_mask) << 8;
 418        limit = (io_limit_lo & io_mask) << 8;
 419
 420        if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
 421                u16 io_base_hi, io_limit_hi;
 422
 423                pci_read_config_word(dev, PCI_IO_BASE_UPPER16, &io_base_hi);
 424                pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, &io_limit_hi);
 425                base |= ((unsigned long) io_base_hi << 16);
 426                limit |= ((unsigned long) io_limit_hi << 16);
 427        }
 428
 429        if (base <= limit) {
 430                res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
 431                region.start = base;
 432                region.end = limit + io_granularity - 1;
 433                pcibios_bus_to_resource(dev->bus, res, &region);
 434                pci_info(dev, "  bridge window %pR\n", res);
 435        }
 436}
 437
 438static void pci_read_bridge_mmio(struct pci_bus *child)
 439{
 440        struct pci_dev *dev = child->self;
 441        u16 mem_base_lo, mem_limit_lo;
 442        unsigned long base, limit;
 443        struct pci_bus_region region;
 444        struct resource *res;
 445
 446        res = child->resource[1];
 447        pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo);
 448        pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo);
 449        base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
 450        limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
 451        if (base <= limit) {
 452                res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
 453                region.start = base;
 454                region.end = limit + 0xfffff;
 455                pcibios_bus_to_resource(dev->bus, res, &region);
 456                pci_info(dev, "  bridge window %pR\n", res);
 457        }
 458}
 459
 460static void pci_read_bridge_mmio_pref(struct pci_bus *child)
 461{
 462        struct pci_dev *dev = child->self;
 463        u16 mem_base_lo, mem_limit_lo;
 464        u64 base64, limit64;
 465        pci_bus_addr_t base, limit;
 466        struct pci_bus_region region;
 467        struct resource *res;
 468
 469        res = child->resource[2];
 470        pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo);
 471        pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo);
 472        base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
 473        limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
 474
 475        if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
 476                u32 mem_base_hi, mem_limit_hi;
 477
 478                pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, &mem_base_hi);
 479                pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, &mem_limit_hi);
 480
 481                /*
 482                 * Some bridges set the base > limit by default, and some
 483                 * (broken) BIOSes do not initialize them.  If we find
 484                 * this, just assume they are not being used.
 485                 */
 486                if (mem_base_hi <= mem_limit_hi) {
 487                        base64 |= (u64) mem_base_hi << 32;
 488                        limit64 |= (u64) mem_limit_hi << 32;
 489                }
 490        }
 491
 492        base = (pci_bus_addr_t) base64;
 493        limit = (pci_bus_addr_t) limit64;
 494
 495        if (base != base64) {
 496                pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
 497                        (unsigned long long) base64);
 498                return;
 499        }
 500
 501        if (base <= limit) {
 502                res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) |
 503                                         IORESOURCE_MEM | IORESOURCE_PREFETCH;
 504                if (res->flags & PCI_PREF_RANGE_TYPE_64)
 505                        res->flags |= IORESOURCE_MEM_64;
 506                region.start = base;
 507                region.end = limit + 0xfffff;
 508                pcibios_bus_to_resource(dev->bus, res, &region);
 509                pci_info(dev, "  bridge window %pR\n", res);
 510        }
 511}
 512
 513void pci_read_bridge_bases(struct pci_bus *child)
 514{
 515        struct pci_dev *dev = child->self;
 516        struct resource *res;
 517        int i;
 518
 519        if (pci_is_root_bus(child))     /* It's a host bus, nothing to read */
 520                return;
 521
 522        pci_info(dev, "PCI bridge to %pR%s\n",
 523                 &child->busn_res,
 524                 dev->transparent ? " (subtractive decode)" : "");
 525
 526        pci_bus_remove_resources(child);
 527        for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
 528                child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
 529
 530        pci_read_bridge_io(child);
 531        pci_read_bridge_mmio(child);
 532        pci_read_bridge_mmio_pref(child);
 533
 534        if (dev->transparent) {
 535                pci_bus_for_each_resource(child->parent, res, i) {
 536                        if (res && res->flags) {
 537                                pci_bus_add_resource(child, res,
 538                                                     PCI_SUBTRACTIVE_DECODE);
 539                                pci_info(dev, "  bridge window %pR (subtractive decode)\n",
 540                                           res);
 541                        }
 542                }
 543        }
 544}
 545
 546static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
 547{
 548        struct pci_bus *b;
 549
 550        b = kzalloc(sizeof(*b), GFP_KERNEL);
 551        if (!b)
 552                return NULL;
 553
 554        INIT_LIST_HEAD(&b->node);
 555        INIT_LIST_HEAD(&b->children);
 556        INIT_LIST_HEAD(&b->devices);
 557        INIT_LIST_HEAD(&b->slots);
 558        INIT_LIST_HEAD(&b->resources);
 559        b->max_bus_speed = PCI_SPEED_UNKNOWN;
 560        b->cur_bus_speed = PCI_SPEED_UNKNOWN;
 561#ifdef CONFIG_PCI_DOMAINS_GENERIC
 562        if (parent)
 563                b->domain_nr = parent->domain_nr;
 564#endif
 565        return b;
 566}
 567
 568static void pci_release_host_bridge_dev(struct device *dev)
 569{
 570        struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
 571
 572        if (bridge->release_fn)
 573                bridge->release_fn(bridge);
 574
 575        pci_free_resource_list(&bridge->windows);
 576        pci_free_resource_list(&bridge->dma_ranges);
 577        kfree(bridge);
 578}
 579
 580static void pci_init_host_bridge(struct pci_host_bridge *bridge)
 581{
 582        INIT_LIST_HEAD(&bridge->windows);
 583        INIT_LIST_HEAD(&bridge->dma_ranges);
 584
 585        /*
 586         * We assume we can manage these PCIe features.  Some systems may
 587         * reserve these for use by the platform itself, e.g., an ACPI BIOS
 588         * may implement its own AER handling and use _OSC to prevent the
 589         * OS from interfering.
 590         */
 591        bridge->native_aer = 1;
 592        bridge->native_pcie_hotplug = 1;
 593        bridge->native_shpc_hotplug = 1;
 594        bridge->native_pme = 1;
 595        bridge->native_ltr = 1;
 596        bridge->native_dpc = 1;
 597
 598        device_initialize(&bridge->dev);
 599}
 600
 601struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 602{
 603        struct pci_host_bridge *bridge;
 604
 605        bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
 606        if (!bridge)
 607                return NULL;
 608
 609        pci_init_host_bridge(bridge);
 610        bridge->dev.release = pci_release_host_bridge_dev;
 611
 612        return bridge;
 613}
 614EXPORT_SYMBOL(pci_alloc_host_bridge);
 615
 616static void devm_pci_alloc_host_bridge_release(void *data)
 617{
 618        pci_free_host_bridge(data);
 619}
 620
 621struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
 622                                                   size_t priv)
 623{
 624        int ret;
 625        struct pci_host_bridge *bridge;
 626
 627        bridge = pci_alloc_host_bridge(priv);
 628        if (!bridge)
 629                return NULL;
 630
 631        bridge->dev.parent = dev;
 632
 633        ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
 634                                       bridge);
 635        if (ret)
 636                return NULL;
 637
 638        ret = devm_of_pci_bridge_init(dev, bridge);
 639        if (ret)
 640                return NULL;
 641
 642        return bridge;
 643}
 644EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
 645
 646void pci_free_host_bridge(struct pci_host_bridge *bridge)
 647{
 648        put_device(&bridge->dev);
 649}
 650EXPORT_SYMBOL(pci_free_host_bridge);
 651
 652/* Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) */
 653static const unsigned char pcix_bus_speed[] = {
 654        PCI_SPEED_UNKNOWN,              /* 0 */
 655        PCI_SPEED_66MHz_PCIX,           /* 1 */
 656        PCI_SPEED_100MHz_PCIX,          /* 2 */
 657        PCI_SPEED_133MHz_PCIX,          /* 3 */
 658        PCI_SPEED_UNKNOWN,              /* 4 */
 659        PCI_SPEED_66MHz_PCIX_ECC,       /* 5 */
 660        PCI_SPEED_100MHz_PCIX_ECC,      /* 6 */
 661        PCI_SPEED_133MHz_PCIX_ECC,      /* 7 */
 662        PCI_SPEED_UNKNOWN,              /* 8 */
 663        PCI_SPEED_66MHz_PCIX_266,       /* 9 */
 664        PCI_SPEED_100MHz_PCIX_266,      /* A */
 665        PCI_SPEED_133MHz_PCIX_266,      /* B */
 666        PCI_SPEED_UNKNOWN,              /* C */
 667        PCI_SPEED_66MHz_PCIX_533,       /* D */
 668        PCI_SPEED_100MHz_PCIX_533,      /* E */
 669        PCI_SPEED_133MHz_PCIX_533       /* F */
 670};
 671
 672/* Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS */
 673const unsigned char pcie_link_speed[] = {
 674        PCI_SPEED_UNKNOWN,              /* 0 */
 675        PCIE_SPEED_2_5GT,               /* 1 */
 676        PCIE_SPEED_5_0GT,               /* 2 */
 677        PCIE_SPEED_8_0GT,               /* 3 */
 678        PCIE_SPEED_16_0GT,              /* 4 */
 679        PCIE_SPEED_32_0GT,              /* 5 */
 680        PCIE_SPEED_64_0GT,              /* 6 */
 681        PCI_SPEED_UNKNOWN,              /* 7 */
 682        PCI_SPEED_UNKNOWN,              /* 8 */
 683        PCI_SPEED_UNKNOWN,              /* 9 */
 684        PCI_SPEED_UNKNOWN,              /* A */
 685        PCI_SPEED_UNKNOWN,              /* B */
 686        PCI_SPEED_UNKNOWN,              /* C */
 687        PCI_SPEED_UNKNOWN,              /* D */
 688        PCI_SPEED_UNKNOWN,              /* E */
 689        PCI_SPEED_UNKNOWN               /* F */
 690};
 691EXPORT_SYMBOL_GPL(pcie_link_speed);
 692
 693const char *pci_speed_string(enum pci_bus_speed speed)
 694{
 695        /* Indexed by the pci_bus_speed enum */
 696        static const char *speed_strings[] = {
 697            "33 MHz PCI",               /* 0x00 */
 698            "66 MHz PCI",               /* 0x01 */
 699            "66 MHz PCI-X",             /* 0x02 */
 700            "100 MHz PCI-X",            /* 0x03 */
 701            "133 MHz PCI-X",            /* 0x04 */
 702            NULL,                       /* 0x05 */
 703            NULL,                       /* 0x06 */
 704            NULL,                       /* 0x07 */
 705            NULL,                       /* 0x08 */
 706            "66 MHz PCI-X 266",         /* 0x09 */
 707            "100 MHz PCI-X 266",        /* 0x0a */
 708            "133 MHz PCI-X 266",        /* 0x0b */
 709            "Unknown AGP",              /* 0x0c */
 710            "1x AGP",                   /* 0x0d */
 711            "2x AGP",                   /* 0x0e */
 712            "4x AGP",                   /* 0x0f */
 713            "8x AGP",                   /* 0x10 */
 714            "66 MHz PCI-X 533",         /* 0x11 */
 715            "100 MHz PCI-X 533",        /* 0x12 */
 716            "133 MHz PCI-X 533",        /* 0x13 */
 717            "2.5 GT/s PCIe",            /* 0x14 */
 718            "5.0 GT/s PCIe",            /* 0x15 */
 719            "8.0 GT/s PCIe",            /* 0x16 */
 720            "16.0 GT/s PCIe",           /* 0x17 */
 721            "32.0 GT/s PCIe",           /* 0x18 */
 722            "64.0 GT/s PCIe",           /* 0x19 */
 723        };
 724
 725        if (speed < ARRAY_SIZE(speed_strings))
 726                return speed_strings[speed];
 727        return "Unknown";
 728}
 729EXPORT_SYMBOL_GPL(pci_speed_string);
 730
 731void pcie_update_link_speed(struct pci_bus *bus, u16 linksta)
 732{
 733        bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS];
 734}
 735EXPORT_SYMBOL_GPL(pcie_update_link_speed);
 736
 737static unsigned char agp_speeds[] = {
 738        AGP_UNKNOWN,
 739        AGP_1X,
 740        AGP_2X,
 741        AGP_4X,
 742        AGP_8X
 743};
 744
 745static enum pci_bus_speed agp_speed(int agp3, int agpstat)
 746{
 747        int index = 0;
 748
 749        if (agpstat & 4)
 750                index = 3;
 751        else if (agpstat & 2)
 752                index = 2;
 753        else if (agpstat & 1)
 754                index = 1;
 755        else
 756                goto out;
 757
 758        if (agp3) {
 759                index += 2;
 760                if (index == 5)
 761                        index = 0;
 762        }
 763
 764 out:
 765        return agp_speeds[index];
 766}
 767
 768static void pci_set_bus_speed(struct pci_bus *bus)
 769{
 770        struct pci_dev *bridge = bus->self;
 771        int pos;
 772
 773        pos = pci_find_capability(bridge, PCI_CAP_ID_AGP);
 774        if (!pos)
 775                pos = pci_find_capability(bridge, PCI_CAP_ID_AGP3);
 776        if (pos) {
 777                u32 agpstat, agpcmd;
 778
 779                pci_read_config_dword(bridge, pos + PCI_AGP_STATUS, &agpstat);
 780                bus->max_bus_speed = agp_speed(agpstat & 8, agpstat & 7);
 781
 782                pci_read_config_dword(bridge, pos + PCI_AGP_COMMAND, &agpcmd);
 783                bus->cur_bus_speed = agp_speed(agpstat & 8, agpcmd & 7);
 784        }
 785
 786        pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
 787        if (pos) {
 788                u16 status;
 789                enum pci_bus_speed max;
 790
 791                pci_read_config_word(bridge, pos + PCI_X_BRIDGE_SSTATUS,
 792                                     &status);
 793
 794                if (status & PCI_X_SSTATUS_533MHZ) {
 795                        max = PCI_SPEED_133MHz_PCIX_533;
 796                } else if (status & PCI_X_SSTATUS_266MHZ) {
 797                        max = PCI_SPEED_133MHz_PCIX_266;
 798                } else if (status & PCI_X_SSTATUS_133MHZ) {
 799                        if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
 800                                max = PCI_SPEED_133MHz_PCIX_ECC;
 801                        else
 802                                max = PCI_SPEED_133MHz_PCIX;
 803                } else {
 804                        max = PCI_SPEED_66MHz_PCIX;
 805                }
 806
 807                bus->max_bus_speed = max;
 808                bus->cur_bus_speed = pcix_bus_speed[
 809                        (status & PCI_X_SSTATUS_FREQ) >> 6];
 810
 811                return;
 812        }
 813
 814        if (pci_is_pcie(bridge)) {
 815                u32 linkcap;
 816                u16 linksta;
 817
 818                pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &linkcap);
 819                bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
 820                bridge->link_active_reporting = !!(linkcap & PCI_EXP_LNKCAP_DLLLARC);
 821
 822                pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta);
 823                pcie_update_link_speed(bus, linksta);
 824        }
 825}
 826
 827static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus)
 828{
 829        struct irq_domain *d;
 830
 831        /*
 832         * Any firmware interface that can resolve the msi_domain
 833         * should be called from here.
 834         */
 835        d = pci_host_bridge_of_msi_domain(bus);
 836        if (!d)
 837                d = pci_host_bridge_acpi_msi_domain(bus);
 838
 839#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
 840        /*
 841         * If no IRQ domain was found via the OF tree, try looking it up
 842         * directly through the fwnode_handle.
 843         */
 844        if (!d) {
 845                struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
 846
 847                if (fwnode)
 848                        d = irq_find_matching_fwnode(fwnode,
 849                                                     DOMAIN_BUS_PCI_MSI);
 850        }
 851#endif
 852
 853        return d;
 854}
 855
 856static void pci_set_bus_msi_domain(struct pci_bus *bus)
 857{
 858        struct irq_domain *d;
 859        struct pci_bus *b;
 860
 861        /*
 862         * The bus can be a root bus, a subordinate bus, or a virtual bus
 863         * created by an SR-IOV device.  Walk up to the first bridge device
 864         * found or derive the domain from the host bridge.
 865         */
 866        for (b = bus, d = NULL; !d && !pci_is_root_bus(b); b = b->parent) {
 867                if (b->self)
 868                        d = dev_get_msi_domain(&b->self->dev);
 869        }
 870
 871        if (!d)
 872                d = pci_host_bridge_msi_domain(b);
 873
 874        dev_set_msi_domain(&bus->dev, d);
 875}
 876
 877static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 878{
 879        struct device *parent = bridge->dev.parent;
 880        struct resource_entry *window, *n;
 881        struct pci_bus *bus, *b;
 882        resource_size_t offset;
 883        LIST_HEAD(resources);
 884        struct resource *res;
 885        char addr[64], *fmt;
 886        const char *name;
 887        int err;
 888
 889        bus = pci_alloc_bus(NULL);
 890        if (!bus)
 891                return -ENOMEM;
 892
 893        bridge->bus = bus;
 894
 895        /* Temporarily move resources off the list */
 896        list_splice_init(&bridge->windows, &resources);
 897        bus->sysdata = bridge->sysdata;
 898        bus->ops = bridge->ops;
 899        bus->number = bus->busn_res.start = bridge->busnr;
 900#ifdef CONFIG_PCI_DOMAINS_GENERIC
 901        bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
 902#endif
 903
 904        b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
 905        if (b) {
 906                /* Ignore it if we already got here via a different bridge */
 907                dev_dbg(&b->dev, "bus already known\n");
 908                err = -EEXIST;
 909                goto free;
 910        }
 911
 912        dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
 913                     bridge->busnr);
 914
 915        err = pcibios_root_bridge_prepare(bridge);
 916        if (err)
 917                goto free;
 918
 919        err = device_add(&bridge->dev);
 920        if (err) {
 921                put_device(&bridge->dev);
 922                goto free;
 923        }
 924        bus->bridge = get_device(&bridge->dev);
 925        device_enable_async_suspend(bus->bridge);
 926        pci_set_bus_of_node(bus);
 927        pci_set_bus_msi_domain(bus);
 928        if (bridge->msi_domain && !dev_get_msi_domain(&bus->dev) &&
 929            !pci_host_of_has_msi_map(parent))
 930                bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;
 931
 932        if (!parent)
 933                set_dev_node(bus->bridge, pcibus_to_node(bus));
 934
 935        bus->dev.class = &pcibus_class;
 936        bus->dev.parent = bus->bridge;
 937
 938        dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
 939        name = dev_name(&bus->dev);
 940
 941        err = device_register(&bus->dev);
 942        if (err)
 943                goto unregister;
 944
 945        pcibios_add_bus(bus);
 946
 947        if (bus->ops->add_bus) {
 948                err = bus->ops->add_bus(bus);
 949                if (WARN_ON(err < 0))
 950                        dev_err(&bus->dev, "failed to add bus: %d\n", err);
 951        }
 952
 953        /* Create legacy_io and legacy_mem files for this bus */
 954        pci_create_legacy_files(bus);
 955
 956        if (parent)
 957                dev_info(parent, "PCI host bridge to bus %s\n", name);
 958        else
 959                pr_info("PCI host bridge to bus %s\n", name);
 960
 961        if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)
 962                dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
 963
 964        /* Add initial resources to the bus */
 965        resource_list_for_each_entry_safe(window, n, &resources) {
 966                list_move_tail(&window->node, &bridge->windows);
 967                offset = window->offset;
 968                res = window->res;
 969
 970                if (res->flags & IORESOURCE_BUS)
 971                        pci_bus_insert_busn_res(bus, bus->number, res->end);
 972                else
 973                        pci_bus_add_resource(bus, res, 0);
 974
 975                if (offset) {
 976                        if (resource_type(res) == IORESOURCE_IO)
 977                                fmt = " (bus address [%#06llx-%#06llx])";
 978                        else
 979                                fmt = " (bus address [%#010llx-%#010llx])";
 980
 981                        snprintf(addr, sizeof(addr), fmt,
 982                                 (unsigned long long)(res->start - offset),
 983                                 (unsigned long long)(res->end - offset));
 984                } else
 985                        addr[0] = '\0';
 986
 987                dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
 988        }
 989
 990        down_write(&pci_bus_sem);
 991        list_add_tail(&bus->node, &pci_root_buses);
 992        up_write(&pci_bus_sem);
 993
 994        return 0;
 995
 996unregister:
 997        put_device(&bridge->dev);
 998        device_del(&bridge->dev);
 999
1000free:
1001        kfree(bus);
1002        return err;
1003}
1004
1005static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1006{
1007        int pos;
1008        u32 status;
1009
1010        /*
1011         * If extended config space isn't accessible on a bridge's primary
1012         * bus, we certainly can't access it on the secondary bus.
1013         */
1014        if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1015                return false;
1016
1017        /*
1018         * PCIe Root Ports and switch ports are PCIe on both sides, so if
1019         * extended config space is accessible on the primary, it's also
1020         * accessible on the secondary.
1021         */
1022        if (pci_is_pcie(bridge) &&
1023            (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT ||
1024             pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM ||
1025             pci_pcie_type(bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1026                return true;
1027
1028        /*
1029         * For the other bridge types:
1030         *   - PCI-to-PCI bridges
1031         *   - PCIe-to-PCI/PCI-X forward bridges
1032         *   - PCI/PCI-X-to-PCIe reverse bridges
1033         * extended config space on the secondary side is only accessible
1034         * if the bridge supports PCI-X Mode 2.
1035         */
1036        pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
1037        if (!pos)
1038                return false;
1039
1040        pci_read_config_dword(bridge, pos + PCI_X_STATUS, &status);
1041        return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
1042}
1043
1044static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
1045                                           struct pci_dev *bridge, int busnr)
1046{
1047        struct pci_bus *child;
1048        struct pci_host_bridge *host;
1049        int i;
1050        int ret;
1051
1052        /* Allocate a new bus and inherit stuff from the parent */
1053        child = pci_alloc_bus(parent);
1054        if (!child)
1055                return NULL;
1056
1057        child->parent = parent;
1058        child->sysdata = parent->sysdata;
1059        child->bus_flags = parent->bus_flags;
1060
1061        host = pci_find_host_bridge(parent);
1062        if (host->child_ops)
1063                child->ops = host->child_ops;
1064        else
1065                child->ops = parent->ops;
1066
1067        /*
1068         * Initialize some portions of the bus device, but don't register
1069         * it now as the parent is not properly set up yet.
1070         */
1071        child->dev.class = &pcibus_class;
1072        dev_set_name(&child->dev, "%04x:%02x", pci_domain_nr(child), busnr);
1073
1074        /* Set up the primary, secondary and subordinate bus numbers */
1075        child->number = child->busn_res.start = busnr;
1076        child->primary = parent->busn_res.start;
1077        child->busn_res.end = 0xff;
1078
1079        if (!bridge) {
1080                child->dev.parent = parent->bridge;
1081                goto add_dev;
1082        }
1083
1084        child->self = bridge;
1085        child->bridge = get_device(&bridge->dev);
1086        child->dev.parent = child->bridge;
1087        pci_set_bus_of_node(child);
1088        pci_set_bus_speed(child);
1089
1090        /*
1091         * Check whether extended config space is accessible on the child
1092         * bus.  Note that we currently assume it is always accessible on
1093         * the root bus.
1094         */
1095        if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1096                child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
1097                pci_info(child, "extended config space not accessible\n");
1098        }
1099
1100        /* Set up default resource pointers and names */
1101        for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1102                child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1103                child->resource[i]->name = child->name;
1104        }
1105        bridge->subordinate = child;
1106
1107add_dev:
1108        pci_set_bus_msi_domain(child);
1109        ret = device_register(&child->dev);
1110        WARN_ON(ret < 0);
1111
1112        pcibios_add_bus(child);
1113
1114        if (child->ops->add_bus) {
1115                ret = child->ops->add_bus(child);
1116                if (WARN_ON(ret < 0))
1117                        dev_err(&child->dev, "failed to add bus: %d\n", ret);
1118        }
1119
1120        /* Create legacy_io and legacy_mem files for this bus */
1121        pci_create_legacy_files(child);
1122
1123        return child;
1124}
1125
1126struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
1127                                int busnr)
1128{
1129        struct pci_bus *child;
1130
1131        child = pci_alloc_child_bus(parent, dev, busnr);
1132        if (child) {
1133                down_write(&pci_bus_sem);
1134                list_add_tail(&child->node, &parent->children);
1135                up_write(&pci_bus_sem);
1136        }
1137        return child;
1138}
1139EXPORT_SYMBOL(pci_add_new_bus);
1140
1141static void pci_enable_crs(struct pci_dev *pdev)
1142{
1143        u16 root_cap = 0;
1144
1145        /* Enable CRS Software Visibility if supported */
1146        pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap);
1147        if (root_cap & PCI_EXP_RTCAP_CRSVIS)
1148                pcie_capability_set_word(pdev, PCI_EXP_RTCTL,
1149                                         PCI_EXP_RTCTL_CRSSVE);
1150}
1151
1152static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1153                                              unsigned int available_buses);
1154/**
1155 * pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1156 * numbers from EA capability.
1157 * @dev: Bridge
1158 * @sec: updated with secondary bus number from EA
1159 * @sub: updated with subordinate bus number from EA
1160 *
1161 * If @dev is a bridge with EA capability that specifies valid secondary
1162 * and subordinate bus numbers, return true with the bus numbers in @sec
1163 * and @sub.  Otherwise return false.
1164 */
1165static bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub)
1166{
1167        int ea, offset;
1168        u32 dw;
1169        u8 ea_sec, ea_sub;
1170
1171        if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1172                return false;
1173
1174        /* find PCI EA capability in list */
1175        ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1176        if (!ea)
1177                return false;
1178
1179        offset = ea + PCI_EA_FIRST_ENT;
1180        pci_read_config_dword(dev, offset, &dw);
1181        ea_sec =  dw & PCI_EA_SEC_BUS_MASK;
1182        ea_sub = (dw & PCI_EA_SUB_BUS_MASK) >> PCI_EA_SUB_BUS_SHIFT;
1183        if (ea_sec  == 0 || ea_sub < ea_sec)
1184                return false;
1185
1186        *sec = ea_sec;
1187        *sub = ea_sub;
1188        return true;
1189}
1190
1191/*
1192 * pci_scan_bridge_extend() - Scan buses behind a bridge
1193 * @bus: Parent bus the bridge is on
1194 * @dev: Bridge itself
1195 * @max: Starting subordinate number of buses behind this bridge
1196 * @available_buses: Total number of buses available for this bridge and
1197 *                   the devices below. After the minimal bus space has
1198 *                   been allocated the remaining buses will be
1199 *                   distributed equally between hotplug-capable bridges.
1200 * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1201 *        that need to be reconfigured.
1202 *
1203 * If it's a bridge, configure it and scan the bus behind it.
1204 * For CardBus bridges, we don't scan behind as the devices will
1205 * be handled by the bridge driver itself.
1206 *
1207 * We need to process bridges in two passes -- first we scan those
1208 * already configured by the BIOS and after we are done with all of
1209 * them, we proceed to assigning numbers to the remaining buses in
1210 * order to avoid overlaps between old and new bus numbers.
1211 *
1212 * Return: New subordinate number covering all buses behind this bridge.
1213 */
1214static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
1215                                  int max, unsigned int available_buses,
1216                                  int pass)
1217{
1218        struct pci_bus *child;
1219        int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
1220        u32 buses, i, j = 0;
1221        u16 bctl;
1222        u8 primary, secondary, subordinate;
1223        int broken = 0;
1224        bool fixed_buses;
1225        u8 fixed_sec, fixed_sub;
1226        int next_busnr;
1227
1228        /*
1229         * Make sure the bridge is powered on to be able to access config
1230         * space of devices below it.
1231         */
1232        pm_runtime_get_sync(&dev->dev);
1233
1234        pci_read_config_dword(dev, PCI_PRIMARY_BUS, &buses);
1235        primary = buses & 0xFF;
1236        secondary = (buses >> 8) & 0xFF;
1237        subordinate = (buses >> 16) & 0xFF;
1238
1239        pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1240                secondary, subordinate, pass);
1241
1242        if (!primary && (primary != bus->number) && secondary && subordinate) {
1243                pci_warn(dev, "Primary bus is hard wired to 0\n");
1244                primary = bus->number;
1245        }
1246
1247        /* Check if setup is sensible at all */
1248        if (!pass &&
1249            (primary != bus->number || secondary <= bus->number ||
1250             secondary > subordinate)) {
1251                pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1252                         secondary, subordinate);
1253                broken = 1;
1254        }
1255
1256        /*
1257         * Disable Master-Abort Mode during probing to avoid reporting of
1258         * bus errors in some architectures.
1259         */
1260        pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &bctl);
1261        pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1262                              bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1263
1264        pci_enable_crs(dev);
1265
1266        if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
1267            !is_cardbus && !broken) {
1268                unsigned int cmax;
1269
1270                /*
1271                 * Bus already configured by firmware, process it in the
1272                 * first pass and just note the configuration.
1273                 */
1274                if (pass)
1275                        goto out;
1276
1277                /*
1278                 * The bus might already exist for two reasons: Either we
1279                 * are rescanning the bus or the bus is reachable through
1280                 * more than one bridge. The second case can happen with
1281                 * the i450NX chipset.
1282                 */
1283                child = pci_find_bus(pci_domain_nr(bus), secondary);
1284                if (!child) {
1285                        child = pci_add_new_bus(bus, dev, secondary);
1286                        if (!child)
1287                                goto out;
1288                        child->primary = primary;
1289                        pci_bus_insert_busn_res(child, secondary, subordinate);
1290                        child->bridge_ctl = bctl;
1291                }
1292
1293                cmax = pci_scan_child_bus(child);
1294                if (cmax > subordinate)
1295                        pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1296                                 subordinate, cmax);
1297
1298                /* Subordinate should equal child->busn_res.end */
1299                if (subordinate > max)
1300                        max = subordinate;
1301        } else {
1302
1303                /*
1304                 * We need to assign a number to this bus which we always
1305                 * do in the second pass.
1306                 */
1307                if (!pass) {
1308                        if (pcibios_assign_all_busses() || broken || is_cardbus)
1309
1310                                /*
1311                                 * Temporarily disable forwarding of the
1312                                 * configuration cycles on all bridges in
1313                                 * this bus segment to avoid possible
1314                                 * conflicts in the second pass between two
1315                                 * bridges programmed with overlapping bus
1316                                 * ranges.
1317                                 */
1318                                pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1319                                                       buses & ~0xffffff);
1320                        goto out;
1321                }
1322
1323                /* Clear errors */
1324                pci_write_config_word(dev, PCI_STATUS, 0xffff);
1325
1326                /* Read bus numbers from EA Capability (if present) */
1327                fixed_buses = pci_ea_fixed_busnrs(dev, &fixed_sec, &fixed_sub);
1328                if (fixed_buses)
1329                        next_busnr = fixed_sec;
1330                else
1331                        next_busnr = max + 1;
1332
1333                /*
1334                 * Prevent assigning a bus number that already exists.
1335                 * This can happen when a bridge is hot-plugged, so in this
1336                 * case we only re-scan this bus.
1337                 */
1338                child = pci_find_bus(pci_domain_nr(bus), next_busnr);
1339                if (!child) {
1340                        child = pci_add_new_bus(bus, dev, next_busnr);
1341                        if (!child)
1342                                goto out;
1343                        pci_bus_insert_busn_res(child, next_busnr,
1344                                                bus->busn_res.end);
1345                }
1346                max++;
1347                if (available_buses)
1348                        available_buses--;
1349
1350                buses = (buses & 0xff000000)
1351                      | ((unsigned int)(child->primary)     <<  0)
1352                      | ((unsigned int)(child->busn_res.start)   <<  8)
1353                      | ((unsigned int)(child->busn_res.end) << 16);
1354
1355                /*
1356                 * yenta.c forces a secondary latency timer of 176.
1357                 * Copy that behaviour here.
1358                 */
1359                if (is_cardbus) {
1360                        buses &= ~0xff000000;
1361                        buses |= CARDBUS_LATENCY_TIMER << 24;
1362                }
1363
1364                /* We need to blast all three values with a single write */
1365                pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
1366
1367                if (!is_cardbus) {
1368                        child->bridge_ctl = bctl;
1369                        max = pci_scan_child_bus_extend(child, available_buses);
1370                } else {
1371
1372                        /*
1373                         * For CardBus bridges, we leave 4 bus numbers as
1374                         * cards with a PCI-to-PCI bridge can be inserted
1375                         * later.
1376                         */
1377                        for (i = 0; i < CARDBUS_RESERVE_BUSNR; i++) {
1378                                struct pci_bus *parent = bus;
1379                                if (pci_find_bus(pci_domain_nr(bus),
1380                                                        max+i+1))
1381                                        break;
1382                                while (parent->parent) {
1383                                        if ((!pcibios_assign_all_busses()) &&
1384                                            (parent->busn_res.end > max) &&
1385                                            (parent->busn_res.end <= max+i)) {
1386                                                j = 1;
1387                                        }
1388                                        parent = parent->parent;
1389                                }
1390                                if (j) {
1391
1392                                        /*
1393                                         * Often, there are two CardBus
1394                                         * bridges -- try to leave one
1395                                         * valid bus number for each one.
1396                                         */
1397                                        i /= 2;
1398                                        break;
1399                                }
1400                        }
1401                        max += i;
1402                }
1403
1404                /*
1405                 * Set subordinate bus number to its real value.
1406                 * If fixed subordinate bus number exists from EA
1407                 * capability then use it.
1408                 */
1409                if (fixed_buses)
1410                        max = fixed_sub;
1411                pci_bus_update_busn_res_end(child, max);
1412                pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, max);
1413        }
1414
1415        sprintf(child->name,
1416                (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
1417                pci_domain_nr(bus), child->number);
1418
1419        /* Check that all devices are accessible */
1420        while (bus->parent) {
1421                if ((child->busn_res.end > bus->busn_res.end) ||
1422                    (child->number > bus->busn_res.end) ||
1423                    (child->number < bus->number) ||
1424                    (child->busn_res.end < bus->number)) {
1425                        dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1426                                 &child->busn_res);
1427                        break;
1428                }
1429                bus = bus->parent;
1430        }
1431
1432out:
1433        pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl);
1434
1435        pm_runtime_put(&dev->dev);
1436
1437        return max;
1438}
1439
1440/*
1441 * pci_scan_bridge() - Scan buses behind a bridge
1442 * @bus: Parent bus the bridge is on
1443 * @dev: Bridge itself
1444 * @max: Starting subordinate number of buses behind this bridge
1445 * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1446 *        that need to be reconfigured.
1447 *
1448 * If it's a bridge, configure it and scan the bus behind it.
1449 * For CardBus bridges, we don't scan behind as the devices will
1450 * be handled by the bridge driver itself.
1451 *
1452 * We need to process bridges in two passes -- first we scan those
1453 * already configured by the BIOS and after we are done with all of
1454 * them, we proceed to assigning numbers to the remaining buses in
1455 * order to avoid overlaps between old and new bus numbers.
1456 *
1457 * Return: New subordinate number covering all buses behind this bridge.
1458 */
1459int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
1460{
1461        return pci_scan_bridge_extend(bus, dev, max, 0, pass);
1462}
1463EXPORT_SYMBOL(pci_scan_bridge);
1464
1465/*
1466 * Read interrupt line and base address registers.
1467 * The architecture-dependent code can tweak these, of course.
1468 */
1469static void pci_read_irq(struct pci_dev *dev)
1470{
1471        unsigned char irq;
1472
1473        /* VFs are not allowed to use INTx, so skip the config reads */
1474        if (dev->is_virtfn) {
1475                dev->pin = 0;
1476                dev->irq = 0;
1477                return;
1478        }
1479
1480        pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irq);
1481        dev->pin = irq;
1482        if (irq)
1483                pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
1484        dev->irq = irq;
1485}
1486
1487void set_pcie_port_type(struct pci_dev *pdev)
1488{
1489        int pos;
1490        u16 reg16;
1491        int type;
1492        struct pci_dev *parent;
1493
1494        pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
1495        if (!pos)
1496                return;
1497
1498        pdev->pcie_cap = pos;
1499        pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, &reg16);
1500        pdev->pcie_flags_reg = reg16;
1501        pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, &reg16);
1502        pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD;
1503
1504        parent = pci_upstream_bridge(pdev);
1505        if (!parent)
1506                return;
1507
1508        /*
1509         * Some systems do not identify their upstream/downstream ports
1510         * correctly so detect impossible configurations here and correct
1511         * the port type accordingly.
1512         */
1513        type = pci_pcie_type(pdev);
1514        if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1515                /*
1516                 * If pdev claims to be downstream port but the parent
1517                 * device is also downstream port assume pdev is actually
1518                 * upstream port.
1519                 */
1520                if (pcie_downstream_port(parent)) {
1521                        pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1522                        pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1523                        pdev->pcie_flags_reg |= PCI_EXP_TYPE_UPSTREAM;
1524                }
1525        } else if (type == PCI_EXP_TYPE_UPSTREAM) {
1526                /*
1527                 * If pdev claims to be upstream port but the parent
1528                 * device is also upstream port assume pdev is actually
1529                 * downstream port.
1530                 */
1531                if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM) {
1532                        pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1533                        pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1534                        pdev->pcie_flags_reg |= PCI_EXP_TYPE_DOWNSTREAM;
1535                }
1536        }
1537}
1538
1539void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1540{
1541        u32 reg32;
1542
1543        pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &reg32);
1544        if (reg32 & PCI_EXP_SLTCAP_HPC)
1545                pdev->is_hotplug_bridge = 1;
1546}
1547
1548static void set_pcie_thunderbolt(struct pci_dev *dev)
1549{
1550        int vsec = 0;
1551        u32 header;
1552
1553        while ((vsec = pci_find_next_ext_capability(dev, vsec,
1554                                                    PCI_EXT_CAP_ID_VNDR))) {
1555                pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header);
1556
1557                /* Is the device part of a Thunderbolt controller? */
1558                if (dev->vendor == PCI_VENDOR_ID_INTEL &&
1559                    PCI_VNDR_HEADER_ID(header) == PCI_VSEC_ID_INTEL_TBT) {
1560                        dev->is_thunderbolt = 1;
1561                        return;
1562                }
1563        }
1564}
1565
1566static void set_pcie_untrusted(struct pci_dev *dev)
1567{
1568        struct pci_dev *parent;
1569
1570        /*
1571         * If the upstream bridge is untrusted we treat this device
1572         * untrusted as well.
1573         */
1574        parent = pci_upstream_bridge(dev);
1575        if (parent && (parent->untrusted || parent->external_facing))
1576                dev->untrusted = true;
1577}
1578
1579static void pci_set_removable(struct pci_dev *dev)
1580{
1581        struct pci_dev *parent = pci_upstream_bridge(dev);
1582
1583        /*
1584         * We (only) consider everything downstream from an external_facing
1585         * device to be removable by the user. We're mainly concerned with
1586         * consumer platforms with user accessible thunderbolt ports that are
1587         * vulnerable to DMA attacks, and we expect those ports to be marked by
1588         * the firmware as external_facing. Devices in traditional hotplug
1589         * slots can technically be removed, but the expectation is that unless
1590         * the port is marked with external_facing, such devices are less
1591         * accessible to user / may not be removed by end user, and thus not
1592         * exposed as "removable" to userspace.
1593         */
1594        if (parent &&
1595            (parent->external_facing || dev_is_removable(&parent->dev)))
1596                dev_set_removable(&dev->dev, DEVICE_REMOVABLE);
1597}
1598
1599/**
1600 * pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1601 * @dev: PCI device
1602 *
1603 * PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1604 * when forwarding a type1 configuration request the bridge must check that
1605 * the extended register address field is zero.  The bridge is not permitted
1606 * to forward the transactions and must handle it as an Unsupported Request.
1607 * Some bridges do not follow this rule and simply drop the extended register
1608 * bits, resulting in the standard config space being aliased, every 256
1609 * bytes across the entire configuration space.  Test for this condition by
1610 * comparing the first dword of each potential alias to the vendor/device ID.
1611 * Known offenders:
1612 *   ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1613 *   AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1614 */
1615static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1616{
1617#ifdef CONFIG_PCI_QUIRKS
1618        int pos;
1619        u32 header, tmp;
1620
1621        pci_read_config_dword(dev, PCI_VENDOR_ID, &header);
1622
1623        for (pos = PCI_CFG_SPACE_SIZE;
1624             pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1625                if (pci_read_config_dword(dev, pos, &tmp) != PCIBIOS_SUCCESSFUL
1626                    || header != tmp)
1627                        return false;
1628        }
1629
1630        return true;
1631#else
1632        return false;
1633#endif
1634}
1635
1636/**
1637 * pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1638 * @dev: PCI device
1639 *
1640 * Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1641 * have 4096 bytes.  Even if the device is capable, that doesn't mean we can
1642 * access it.  Maybe we don't have a way to generate extended config space
1643 * accesses, or the device is behind a reverse Express bridge.  So we try
1644 * reading the dword at 0x100 which must either be 0 or a valid extended
1645 * capability header.
1646 */
1647static int pci_cfg_space_size_ext(struct pci_dev *dev)
1648{
1649        u32 status;
1650        int pos = PCI_CFG_SPACE_SIZE;
1651
1652        if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL)
1653                return PCI_CFG_SPACE_SIZE;
1654        if (status == 0xffffffff || pci_ext_cfg_is_aliased(dev))
1655                return PCI_CFG_SPACE_SIZE;
1656
1657        return PCI_CFG_SPACE_EXP_SIZE;
1658}
1659
1660int pci_cfg_space_size(struct pci_dev *dev)
1661{
1662        int pos;
1663        u32 status;
1664        u16 class;
1665
1666#ifdef CONFIG_PCI_IOV
1667        /*
1668         * Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1669         * implement a PCIe capability and therefore must implement extended
1670         * config space.  We can skip the NO_EXTCFG test below and the
1671         * reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1672         * the fact that the SR-IOV capability on the PF resides in extended
1673         * config space and must be accessible and non-aliased to have enabled
1674         * support for this VF.  This is a micro performance optimization for
1675         * systems supporting many VFs.
1676         */
1677        if (dev->is_virtfn)
1678                return PCI_CFG_SPACE_EXP_SIZE;
1679#endif
1680
1681        if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1682                return PCI_CFG_SPACE_SIZE;
1683
1684        class = dev->class >> 8;
1685        if (class == PCI_CLASS_BRIDGE_HOST)
1686                return pci_cfg_space_size_ext(dev);
1687
1688        if (pci_is_pcie(dev))
1689                return pci_cfg_space_size_ext(dev);
1690
1691        pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1692        if (!pos)
1693                return PCI_CFG_SPACE_SIZE;
1694
1695        pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
1696        if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
1697                return pci_cfg_space_size_ext(dev);
1698
1699        return PCI_CFG_SPACE_SIZE;
1700}
1701
1702static u32 pci_class(struct pci_dev *dev)
1703{
1704        u32 class;
1705
1706#ifdef CONFIG_PCI_IOV
1707        if (dev->is_virtfn)
1708                return dev->physfn->sriov->class;
1709#endif
1710        pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
1711        return class;
1712}
1713
1714static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
1715{
1716#ifdef CONFIG_PCI_IOV
1717        if (dev->is_virtfn) {
1718                *vendor = dev->physfn->sriov->subsystem_vendor;
1719                *device = dev->physfn->sriov->subsystem_device;
1720                return;
1721        }
1722#endif
1723        pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, vendor);
1724        pci_read_config_word(dev, PCI_SUBSYSTEM_ID, device);
1725}
1726
1727static u8 pci_hdr_type(struct pci_dev *dev)
1728{
1729        u8 hdr_type;
1730
1731#ifdef CONFIG_PCI_IOV
1732        if (dev->is_virtfn)
1733                return dev->physfn->sriov->hdr_type;
1734#endif
1735        pci_read_config_byte(dev, PCI_HEADER_TYPE, &hdr_type);
1736        return hdr_type;
1737}
1738
1739#define LEGACY_IO_RESOURCE      (IORESOURCE_IO | IORESOURCE_PCI_FIXED)
1740
1741/**
1742 * pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1743 * @dev: PCI device
1744 *
1745 * Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev.  Check this
1746 * at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1747 */
1748static int pci_intx_mask_broken(struct pci_dev *dev)
1749{
1750        u16 orig, toggle, new;
1751
1752        pci_read_config_word(dev, PCI_COMMAND, &orig);
1753        toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1754        pci_write_config_word(dev, PCI_COMMAND, toggle);
1755        pci_read_config_word(dev, PCI_COMMAND, &new);
1756
1757        pci_write_config_word(dev, PCI_COMMAND, orig);
1758
1759        /*
1760         * PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1761         * r2.3, so strictly speaking, a device is not *broken* if it's not
1762         * writable.  But we'll live with the misnomer for now.
1763         */
1764        if (new != toggle)
1765                return 1;
1766        return 0;
1767}
1768
1769static void early_dump_pci_device(struct pci_dev *pdev)
1770{
1771        u32 value[256 / 4];
1772        int i;
1773
1774        pci_info(pdev, "config space:\n");
1775
1776        for (i = 0; i < 256; i += 4)
1777                pci_read_config_dword(pdev, i, &value[i / 4]);
1778
1779        print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
1780                       value, 256, false);
1781}
1782
1783/**
1784 * pci_setup_device - Fill in class and map information of a device
1785 * @dev: the device structure to fill
1786 *
1787 * Initialize the device structure with information about the device's
1788 * vendor,class,memory and IO-space addresses, IRQ lines etc.
1789 * Called at initialisation of the PCI subsystem and by CardBus services.
1790 * Returns 0 on success and negative if unknown type of device (not normal,
1791 * bridge or CardBus).
1792 */
1793int pci_setup_device(struct pci_dev *dev)
1794{
1795        u32 class;
1796        u16 cmd;
1797        u8 hdr_type;
1798        int pos = 0;
1799        struct pci_bus_region region;
1800        struct resource *res;
1801
1802        hdr_type = pci_hdr_type(dev);
1803
1804        dev->sysdata = dev->bus->sysdata;
1805        dev->dev.parent = dev->bus->bridge;
1806        dev->dev.bus = &pci_bus_type;
1807        dev->hdr_type = hdr_type & 0x7f;
1808        dev->multifunction = !!(hdr_type & 0x80);
1809        dev->error_state = pci_channel_io_normal;
1810        set_pcie_port_type(dev);
1811
1812        pci_dev_assign_slot(dev);
1813
1814        /*
1815         * Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
1816         * set this higher, assuming the system even supports it.
1817         */
1818        dev->dma_mask = 0xffffffff;
1819
1820        dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(dev->bus),
1821                     dev->bus->number, PCI_SLOT(dev->devfn),
1822                     PCI_FUNC(dev->devfn));
1823
1824        class = pci_class(dev);
1825
1826        dev->revision = class & 0xff;
1827        dev->class = class >> 8;                    /* upper 3 bytes */
1828
1829        if (pci_early_dump)
1830                early_dump_pci_device(dev);
1831
1832        /* Need to have dev->class ready */
1833        dev->cfg_size = pci_cfg_space_size(dev);
1834
1835        /* Need to have dev->cfg_size ready */
1836        set_pcie_thunderbolt(dev);
1837
1838        set_pcie_untrusted(dev);
1839
1840        /* "Unknown power state" */
1841        dev->current_state = PCI_UNKNOWN;
1842
1843        /* Early fixups, before probing the BARs */
1844        pci_fixup_device(pci_fixup_early, dev);
1845
1846        pci_set_removable(dev);
1847
1848        pci_info(dev, "[%04x:%04x] type %02x class %#08x\n",
1849                 dev->vendor, dev->device, dev->hdr_type, dev->class);
1850
1851        /* Device class may be changed after fixup */
1852        class = dev->class >> 8;
1853
1854        if (dev->non_compliant_bars && !dev->mmio_always_on) {
1855                pci_read_config_word(dev, PCI_COMMAND, &cmd);
1856                if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) {
1857                        pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
1858                        cmd &= ~PCI_COMMAND_IO;
1859                        cmd &= ~PCI_COMMAND_MEMORY;
1860                        pci_write_config_word(dev, PCI_COMMAND, cmd);
1861                }
1862        }
1863
1864        dev->broken_intx_masking = pci_intx_mask_broken(dev);
1865
1866        switch (dev->hdr_type) {                    /* header type */
1867        case PCI_HEADER_TYPE_NORMAL:                /* standard header */
1868                if (class == PCI_CLASS_BRIDGE_PCI)
1869                        goto bad;
1870                pci_read_irq(dev);
1871                pci_read_bases(dev, 6, PCI_ROM_ADDRESS);
1872
1873                pci_subsystem_ids(dev, &dev->subsystem_vendor, &dev->subsystem_device);
1874
1875                /*
1876                 * Do the ugly legacy mode stuff here rather than broken chip
1877                 * quirk code. Legacy mode ATA controllers have fixed
1878                 * addresses. These are not always echoed in BAR0-3, and
1879                 * BAR0-3 in a few cases contain junk!
1880                 */
1881                if (class == PCI_CLASS_STORAGE_IDE) {
1882                        u8 progif;
1883                        pci_read_config_byte(dev, PCI_CLASS_PROG, &progif);
1884                        if ((progif & 1) == 0) {
1885                                region.start = 0x1F0;
1886                                region.end = 0x1F7;
1887                                res = &dev->resource[0];
1888                                res->flags = LEGACY_IO_RESOURCE;
1889                                pcibios_bus_to_resource(dev->bus, res, &region);
1890                                pci_info(dev, "legacy IDE quirk: reg 0x10: %pR\n",
1891                                         res);
1892                                region.start = 0x3F6;
1893                                region.end = 0x3F6;
1894                                res = &dev->resource[1];
1895                                res->flags = LEGACY_IO_RESOURCE;
1896                                pcibios_bus_to_resource(dev->bus, res, &region);
1897                                pci_info(dev, "legacy IDE quirk: reg 0x14: %pR\n",
1898                                         res);
1899                        }
1900                        if ((progif & 4) == 0) {
1901                                region.start = 0x170;
1902                                region.end = 0x177;
1903                                res = &dev->resource[2];
1904                                res->flags = LEGACY_IO_RESOURCE;
1905                                pcibios_bus_to_resource(dev->bus, res, &region);
1906                                pci_info(dev, "legacy IDE quirk: reg 0x18: %pR\n",
1907                                         res);
1908                                region.start = 0x376;
1909                                region.end = 0x376;
1910                                res = &dev->resource[3];
1911                                res->flags = LEGACY_IO_RESOURCE;
1912                                pcibios_bus_to_resource(dev->bus, res, &region);
1913                                pci_info(dev, "legacy IDE quirk: reg 0x1c: %pR\n",
1914                                         res);
1915                        }
1916                }
1917                break;
1918
1919        case PCI_HEADER_TYPE_BRIDGE:                /* bridge header */
1920                /*
1921                 * The PCI-to-PCI bridge spec requires that subtractive
1922                 * decoding (i.e. transparent) bridge must have programming
1923                 * interface code of 0x01.
1924                 */
1925                pci_read_irq(dev);
1926                dev->transparent = ((dev->class & 0xff) == 1);
1927                pci_read_bases(dev, 2, PCI_ROM_ADDRESS1);
1928                pci_read_bridge_windows(dev);
1929                set_pcie_hotplug_bridge(dev);
1930                pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
1931                if (pos) {
1932                        pci_read_config_word(dev, pos + PCI_SSVID_VENDOR_ID, &dev->subsystem_vendor);
1933                        pci_read_config_word(dev, pos + PCI_SSVID_DEVICE_ID, &dev->subsystem_device);
1934                }
1935                break;
1936
1937        case PCI_HEADER_TYPE_CARDBUS:               /* CardBus bridge header */
1938                if (class != PCI_CLASS_BRIDGE_CARDBUS)
1939                        goto bad;
1940                pci_read_irq(dev);
1941                pci_read_bases(dev, 1, 0);
1942                pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, &dev->subsystem_vendor);
1943                pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, &dev->subsystem_device);
1944                break;
1945
1946        default:                                    /* unknown header */
1947                pci_err(dev, "unknown header type %02x, ignoring device\n",
1948                        dev->hdr_type);
1949                return -EIO;
1950
1951        bad:
1952                pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
1953                        dev->class, dev->hdr_type);
1954                dev->class = PCI_CLASS_NOT_DEFINED << 8;
1955        }
1956
1957        /* We found a fine healthy device, go go go... */
1958        return 0;
1959}
1960
1961static void pci_configure_mps(struct pci_dev *dev)
1962{
1963        struct pci_dev *bridge = pci_upstream_bridge(dev);
1964        int mps, mpss, p_mps, rc;
1965
1966        if (!pci_is_pcie(dev))
1967                return;
1968
1969        /* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
1970        if (dev->is_virtfn)
1971                return;
1972
1973        /*
1974         * For Root Complex Integrated Endpoints, program the maximum
1975         * supported value unless limited by the PCIE_BUS_PEER2PEER case.
1976         */
1977        if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
1978                if (pcie_bus_config == PCIE_BUS_PEER2PEER)
1979                        mps = 128;
1980                else
1981                        mps = 128 << dev->pcie_mpss;
1982                rc = pcie_set_mps(dev, mps);
1983                if (rc) {
1984                        pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
1985                                 mps);
1986                }
1987                return;
1988        }
1989
1990        if (!bridge || !pci_is_pcie(bridge))
1991                return;
1992
1993        mps = pcie_get_mps(dev);
1994        p_mps = pcie_get_mps(bridge);
1995
1996        if (mps == p_mps)
1997                return;
1998
1999        if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2000                pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2001                         mps, pci_name(bridge), p_mps);
2002                return;
2003        }
2004
2005        /*
2006         * Fancier MPS configuration is done later by
2007         * pcie_bus_configure_settings()
2008         */
2009        if (pcie_bus_config != PCIE_BUS_DEFAULT)
2010                return;
2011
2012        mpss = 128 << dev->pcie_mpss;
2013        if (mpss < p_mps && pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2014                pcie_set_mps(bridge, mpss);
2015                pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2016                         mpss, p_mps, 128 << bridge->pcie_mpss);
2017                p_mps = pcie_get_mps(bridge);
2018        }
2019
2020        rc = pcie_set_mps(dev, p_mps);
2021        if (rc) {
2022                pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2023                         p_mps);
2024                return;
2025        }
2026
2027        pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2028                 p_mps, mps, mpss);
2029}
2030
2031int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
2032{
2033        struct pci_host_bridge *host;
2034        u32 cap;
2035        u16 ctl;
2036        int ret;
2037
2038        if (!pci_is_pcie(dev))
2039                return 0;
2040
2041        ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
2042        if (ret)
2043                return 0;
2044
2045        if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2046                return 0;
2047
2048        ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
2049        if (ret)
2050                return 0;
2051
2052        host = pci_find_host_bridge(dev->bus);
2053        if (!host)
2054                return 0;
2055
2056        /*
2057         * If some device in the hierarchy doesn't handle Extended Tags
2058         * correctly, make sure they're disabled.
2059         */
2060        if (host->no_ext_tags) {
2061                if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2062                        pci_info(dev, "disabling Extended Tags\n");
2063                        pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2064                                                   PCI_EXP_DEVCTL_EXT_TAG);
2065                }
2066                return 0;
2067        }
2068
2069        if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2070                pci_info(dev, "enabling Extended Tags\n");
2071                pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2072                                         PCI_EXP_DEVCTL_EXT_TAG);
2073        }
2074        return 0;
2075}
2076
2077/**
2078 * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2079 * @dev: PCI device to query
2080 *
2081 * Returns true if the device has enabled relaxed ordering attribute.
2082 */
2083bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2084{
2085        u16 v;
2086
2087        pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &v);
2088
2089        return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2090}
2091EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2092
2093static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2094{
2095        struct pci_dev *root;
2096
2097        /* PCI_EXP_DEVICE_RELAX_EN is RsvdP in VFs */
2098        if (dev->is_virtfn)
2099                return;
2100
2101        if (!pcie_relaxed_ordering_enabled(dev))
2102                return;
2103
2104        /*
2105         * For now, we only deal with Relaxed Ordering issues with Root
2106         * Ports. Peer-to-Peer DMA is another can of worms.
2107         */
2108        root = pcie_find_root_port(dev);
2109        if (!root)
2110                return;
2111
2112        if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2113                pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2114                                           PCI_EXP_DEVCTL_RELAX_EN);
2115                pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2116        }
2117}
2118
2119static void pci_configure_ltr(struct pci_dev *dev)
2120{
2121#ifdef CONFIG_PCIEASPM
2122        struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
2123        struct pci_dev *bridge;
2124        u32 cap, ctl;
2125
2126        if (!pci_is_pcie(dev))
2127                return;
2128
2129        /* Read L1 PM substate capabilities */
2130        dev->l1ss = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_L1SS);
2131
2132        pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
2133        if (!(cap & PCI_EXP_DEVCAP2_LTR))
2134                return;
2135
2136        pcie_capability_read_dword(dev, PCI_EXP_DEVCTL2, &ctl);
2137        if (ctl & PCI_EXP_DEVCTL2_LTR_EN) {
2138                if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
2139                        dev->ltr_path = 1;
2140                        return;
2141                }
2142
2143                bridge = pci_upstream_bridge(dev);
2144                if (bridge && bridge->ltr_path)
2145                        dev->ltr_path = 1;
2146
2147                return;
2148        }
2149
2150        if (!host->native_ltr)
2151                return;
2152
2153        /*
2154         * Software must not enable LTR in an Endpoint unless the Root
2155         * Complex and all intermediate Switches indicate support for LTR.
2156         * PCIe r4.0, sec 6.18.
2157         */
2158        if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
2159            ((bridge = pci_upstream_bridge(dev)) &&
2160              bridge->ltr_path)) {
2161                pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
2162                                         PCI_EXP_DEVCTL2_LTR_EN);
2163                dev->ltr_path = 1;
2164        }
2165#endif
2166}
2167
2168static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2169{
2170#ifdef CONFIG_PCI_PASID
2171        struct pci_dev *bridge;
2172        int pcie_type;
2173        u32 cap;
2174
2175        if (!pci_is_pcie(dev))
2176                return;
2177
2178        pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
2179        if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2180                return;
2181
2182        pcie_type = pci_pcie_type(dev);
2183        if (pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
2184            pcie_type == PCI_EXP_TYPE_RC_END)
2185                dev->eetlp_prefix_path = 1;
2186        else {
2187                bridge = pci_upstream_bridge(dev);
2188                if (bridge && bridge->eetlp_prefix_path)
2189                        dev->eetlp_prefix_path = 1;
2190        }
2191#endif
2192}
2193
2194static void pci_configure_serr(struct pci_dev *dev)
2195{
2196        u16 control;
2197
2198        if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2199
2200                /*
2201                 * A bridge will not forward ERR_ messages coming from an
2202                 * endpoint unless SERR# forwarding is enabled.
2203                 */
2204                pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &control);
2205                if (!(control & PCI_BRIDGE_CTL_SERR)) {
2206                        control |= PCI_BRIDGE_CTL_SERR;
2207                        pci_write_config_word(dev, PCI_BRIDGE_CONTROL, control);
2208                }
2209        }
2210}
2211
2212static void pci_configure_device(struct pci_dev *dev)
2213{
2214        pci_configure_mps(dev);
2215        pci_configure_extended_tags(dev, NULL);
2216        pci_configure_relaxed_ordering(dev);
2217        pci_configure_ltr(dev);
2218        pci_configure_eetlp_prefix(dev);
2219        pci_configure_serr(dev);
2220
2221        pci_acpi_program_hp_params(dev);
2222}
2223
2224static void pci_release_capabilities(struct pci_dev *dev)
2225{
2226        pci_aer_exit(dev);
2227        pci_rcec_exit(dev);
2228        pci_vpd_release(dev);
2229        pci_iov_release(dev);
2230        pci_free_cap_save_buffers(dev);
2231}
2232
2233/**
2234 * pci_release_dev - Free a PCI device structure when all users of it are
2235 *                   finished
2236 * @dev: device that's been disconnected
2237 *
2238 * Will be called only by the device core when all users of this PCI device are
2239 * done.
2240 */
2241static void pci_release_dev(struct device *dev)
2242{
2243        struct pci_dev *pci_dev;
2244
2245        pci_dev = to_pci_dev(dev);
2246        pci_release_capabilities(pci_dev);
2247        pci_release_of_node(pci_dev);
2248        pcibios_release_device(pci_dev);
2249        pci_bus_put(pci_dev->bus);
2250        kfree(pci_dev->driver_override);
2251        bitmap_free(pci_dev->dma_alias_mask);
2252        dev_dbg(dev, "device released\n");
2253        kfree(pci_dev);
2254}
2255
2256struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
2257{
2258        struct pci_dev *dev;
2259
2260        dev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
2261        if (!dev)
2262                return NULL;
2263
2264        INIT_LIST_HEAD(&dev->bus_list);
2265        dev->dev.type = &pci_dev_type;
2266        dev->bus = pci_bus_get(bus);
2267
2268        return dev;
2269}
2270EXPORT_SYMBOL(pci_alloc_dev);
2271
2272static bool pci_bus_crs_vendor_id(u32 l)
2273{
2274        return (l & 0xffff) == 0x0001;
2275}
2276
2277static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
2278                             int timeout)
2279{
2280        int delay = 1;
2281
2282        if (!pci_bus_crs_vendor_id(*l))
2283                return true;    /* not a CRS completion */
2284
2285        if (!timeout)
2286                return false;   /* CRS, but caller doesn't want to wait */
2287
2288        /*
2289         * We got the reserved Vendor ID that indicates a completion with
2290         * Configuration Request Retry Status (CRS).  Retry until we get a
2291         * valid Vendor ID or we time out.
2292         */
2293        while (pci_bus_crs_vendor_id(*l)) {
2294                if (delay > timeout) {
2295                        pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2296                                pci_domain_nr(bus), bus->number,
2297                                PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2298
2299                        return false;
2300                }
2301                if (delay >= 1000)
2302                        pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2303                                pci_domain_nr(bus), bus->number,
2304                                PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2305
2306                msleep(delay);
2307                delay *= 2;
2308
2309                if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2310                        return false;
2311        }
2312
2313        if (delay >= 1000)
2314                pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2315                        pci_domain_nr(bus), bus->number,
2316                        PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2317
2318        return true;
2319}
2320
2321bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2322                                        int timeout)
2323{
2324        if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2325                return false;
2326
2327        /* Some broken boards return 0 or ~0 if a slot is empty: */
2328        if (*l == 0xffffffff || *l == 0x00000000 ||
2329            *l == 0x0000ffff || *l == 0xffff0000)
2330                return false;
2331
2332        if (pci_bus_crs_vendor_id(*l))
2333                return pci_bus_wait_crs(bus, devfn, l, timeout);
2334
2335        return true;
2336}
2337
2338bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2339                                int timeout)
2340{
2341#ifdef CONFIG_PCI_QUIRKS
2342        struct pci_dev *bridge = bus->self;
2343
2344        /*
2345         * Certain IDT switches have an issue where they improperly trigger
2346         * ACS Source Validation errors on completions for config reads.
2347         */
2348        if (bridge && bridge->vendor == PCI_VENDOR_ID_IDT &&
2349            bridge->device == 0x80b5)
2350                return pci_idt_bus_quirk(bus, devfn, l, timeout);
2351#endif
2352
2353        return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2354}
2355EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2356
2357/*
2358 * Read the config data for a PCI device, sanity-check it,
2359 * and fill in the dev structure.
2360 */
2361static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
2362{
2363        struct pci_dev *dev;
2364        u32 l;
2365
2366        if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000))
2367                return NULL;
2368
2369        dev = pci_alloc_dev(bus);
2370        if (!dev)
2371                return NULL;
2372
2373        dev->devfn = devfn;
2374        dev->vendor = l & 0xffff;
2375        dev->device = (l >> 16) & 0xffff;
2376
2377        pci_set_of_node(dev);
2378
2379        if (pci_setup_device(dev)) {
2380                pci_release_of_node(dev);
2381                pci_bus_put(dev->bus);
2382                kfree(dev);
2383                return NULL;
2384        }
2385
2386        return dev;
2387}
2388
2389void pcie_report_downtraining(struct pci_dev *dev)
2390{
2391        if (!pci_is_pcie(dev))
2392                return;
2393
2394        /* Look from the device up to avoid downstream ports with no devices */
2395        if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2396            (pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2397            (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2398                return;
2399
2400        /* Multi-function PCIe devices share the same link/status */
2401        if (PCI_FUNC(dev->devfn) != 0 || dev->is_virtfn)
2402                return;
2403
2404        /* Print link status only if the device is constrained by the fabric */
2405        __pcie_print_link_status(dev, false);
2406}
2407
2408static void pci_init_capabilities(struct pci_dev *dev)
2409{
2410        pci_ea_init(dev);               /* Enhanced Allocation */
2411        pci_msi_init(dev);              /* Disable MSI */
2412        pci_msix_init(dev);             /* Disable MSI-X */
2413
2414        /* Buffers for saving PCIe and PCI-X capabilities */
2415        pci_allocate_cap_save_buffers(dev);
2416
2417        pci_pm_init(dev);               /* Power Management */
2418        pci_vpd_init(dev);              /* Vital Product Data */
2419        pci_configure_ari(dev);         /* Alternative Routing-ID Forwarding */
2420        pci_iov_init(dev);              /* Single Root I/O Virtualization */
2421        pci_ats_init(dev);              /* Address Translation Services */
2422        pci_pri_init(dev);              /* Page Request Interface */
2423        pci_pasid_init(dev);            /* Process Address Space ID */
2424        pci_acs_init(dev);              /* Access Control Services */
2425        pci_ptm_init(dev);              /* Precision Time Measurement */
2426        pci_aer_init(dev);              /* Advanced Error Reporting */
2427        pci_dpc_init(dev);              /* Downstream Port Containment */
2428        pci_rcec_init(dev);             /* Root Complex Event Collector */
2429
2430        pcie_report_downtraining(dev);
2431
2432        if (pci_probe_reset_function(dev) == 0)
2433                dev->reset_fn = 1;
2434}
2435
2436/*
2437 * This is the equivalent of pci_host_bridge_msi_domain() that acts on
2438 * devices. Firmware interfaces that can select the MSI domain on a
2439 * per-device basis should be called from here.
2440 */
2441static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev)
2442{
2443        struct irq_domain *d;
2444
2445        /*
2446         * If a domain has been set through the pcibios_add_device()
2447         * callback, then this is the one (platform code knows best).
2448         */
2449        d = dev_get_msi_domain(&dev->dev);
2450        if (d)
2451                return d;
2452
2453        /*
2454         * Let's see if we have a firmware interface able to provide
2455         * the domain.
2456         */
2457        d = pci_msi_get_device_domain(dev);
2458        if (d)
2459                return d;
2460
2461        return NULL;
2462}
2463
2464static void pci_set_msi_domain(struct pci_dev *dev)
2465{
2466        struct irq_domain *d;
2467
2468        /*
2469         * If the platform or firmware interfaces cannot supply a
2470         * device-specific MSI domain, then inherit the default domain
2471         * from the host bridge itself.
2472         */
2473        d = pci_dev_msi_domain(dev);
2474        if (!d)
2475                d = dev_get_msi_domain(&dev->bus->dev);
2476
2477        dev_set_msi_domain(&dev->dev, d);
2478}
2479
2480void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
2481{
2482        int ret;
2483
2484        pci_configure_device(dev);
2485
2486        device_initialize(&dev->dev);
2487        dev->dev.release = pci_release_dev;
2488
2489        set_dev_node(&dev->dev, pcibus_to_node(bus));
2490        dev->dev.dma_mask = &dev->dma_mask;
2491        dev->dev.dma_parms = &dev->dma_parms;
2492        dev->dev.coherent_dma_mask = 0xffffffffull;
2493
2494        dma_set_max_seg_size(&dev->dev, 65536);
2495        dma_set_seg_boundary(&dev->dev, 0xffffffff);
2496
2497        /* Fix up broken headers */
2498        pci_fixup_device(pci_fixup_header, dev);
2499
2500        pci_reassigndev_resource_alignment(dev);
2501
2502        dev->state_saved = false;
2503
2504        pci_init_capabilities(dev);
2505
2506        /*
2507         * Add the device to our list of discovered devices
2508         * and the bus list for fixup functions, etc.
2509         */
2510        down_write(&pci_bus_sem);
2511        list_add_tail(&dev->bus_list, &bus->devices);
2512        up_write(&pci_bus_sem);
2513
2514        ret = pcibios_add_device(dev);
2515        WARN_ON(ret < 0);
2516
2517        /* Set up MSI IRQ domain */
2518        pci_set_msi_domain(dev);
2519
2520        /* Notifier could use PCI capabilities */
2521        dev->match_driver = false;
2522        ret = device_add(&dev->dev);
2523        WARN_ON(ret < 0);
2524}
2525
2526struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn)
2527{
2528        struct pci_dev *dev;
2529
2530        dev = pci_get_slot(bus, devfn);
2531        if (dev) {
2532                pci_dev_put(dev);
2533                return dev;
2534        }
2535
2536        dev = pci_scan_device(bus, devfn);
2537        if (!dev)
2538                return NULL;
2539
2540        pci_device_add(dev, bus);
2541
2542        return dev;
2543}
2544EXPORT_SYMBOL(pci_scan_single_device);
2545
2546static unsigned next_fn(struct pci_bus *bus, struct pci_dev *dev, unsigned fn)
2547{
2548        int pos;
2549        u16 cap = 0;
2550        unsigned next_fn;
2551
2552        if (pci_ari_enabled(bus)) {
2553                if (!dev)
2554                        return 0;
2555                pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2556                if (!pos)
2557                        return 0;
2558
2559                pci_read_config_word(dev, pos + PCI_ARI_CAP, &cap);
2560                next_fn = PCI_ARI_CAP_NFN(cap);
2561                if (next_fn <= fn)
2562                        return 0;       /* protect against malformed list */
2563
2564                return next_fn;
2565        }
2566
2567        /* dev may be NULL for non-contiguous multifunction devices */
2568        if (!dev || dev->multifunction)
2569                return (fn + 1) % 8;
2570
2571        return 0;
2572}
2573
2574static int only_one_child(struct pci_bus *bus)
2575{
2576        struct pci_dev *bridge = bus->self;
2577
2578        /*
2579         * Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2580         * we scan for all possible devices, not just Device 0.
2581         */
2582        if (pci_has_flag(PCI_SCAN_ALL_PCIE_DEVS))
2583                return 0;
2584
2585        /*
2586         * A PCIe Downstream Port normally leads to a Link with only Device
2587         * 0 on it (PCIe spec r3.1, sec 7.3.1).  As an optimization, scan
2588         * only for Device 0 in that situation.
2589         */
2590        if (bridge && pci_is_pcie(bridge) && pcie_downstream_port(bridge))
2591                return 1;
2592
2593        return 0;
2594}
2595
2596/**
2597 * pci_scan_slot - Scan a PCI slot on a bus for devices
2598 * @bus: PCI bus to scan
2599 * @devfn: slot number to scan (must have zero function)
2600 *
2601 * Scan a PCI slot on the specified PCI bus for devices, adding
2602 * discovered devices to the @bus->devices list.  New devices
2603 * will not have is_added set.
2604 *
2605 * Returns the number of new devices found.
2606 */
2607int pci_scan_slot(struct pci_bus *bus, int devfn)
2608{
2609        unsigned fn, nr = 0;
2610        struct pci_dev *dev;
2611
2612        if (only_one_child(bus) && (devfn > 0))
2613                return 0; /* Already scanned the entire slot */
2614
2615        dev = pci_scan_single_device(bus, devfn);
2616        if (!dev)
2617                return 0;
2618        if (!pci_dev_is_added(dev))
2619                nr++;
2620
2621        for (fn = next_fn(bus, dev, 0); fn > 0; fn = next_fn(bus, dev, fn)) {
2622                dev = pci_scan_single_device(bus, devfn + fn);
2623                if (dev) {
2624                        if (!pci_dev_is_added(dev))
2625                                nr++;
2626                        dev->multifunction = 1;
2627                }
2628        }
2629
2630        /* Only one slot has PCIe device */
2631        if (bus->self && nr)
2632                pcie_aspm_init_link_state(bus->self);
2633
2634        return nr;
2635}
2636EXPORT_SYMBOL(pci_scan_slot);
2637
2638static int pcie_find_smpss(struct pci_dev *dev, void *data)
2639{
2640        u8 *smpss = data;
2641
2642        if (!pci_is_pcie(dev))
2643                return 0;
2644
2645        /*
2646         * We don't have a way to change MPS settings on devices that have
2647         * drivers attached.  A hot-added device might support only the minimum
2648         * MPS setting (MPS=128).  Therefore, if the fabric contains a bridge
2649         * where devices may be hot-added, we limit the fabric MPS to 128 so
2650         * hot-added devices will work correctly.
2651         *
2652         * However, if we hot-add a device to a slot directly below a Root
2653         * Port, it's impossible for there to be other existing devices below
2654         * the port.  We don't limit the MPS in this case because we can
2655         * reconfigure MPS on both the Root Port and the hot-added device,
2656         * and there are no other devices involved.
2657         *
2658         * Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2659         */
2660        if (dev->is_hotplug_bridge &&
2661            pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2662                *smpss = 0;
2663
2664        if (*smpss > dev->pcie_mpss)
2665                *smpss = dev->pcie_mpss;
2666
2667        return 0;
2668}
2669
2670static void pcie_write_mps(struct pci_dev *dev, int mps)
2671{
2672        int rc;
2673
2674        if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2675                mps = 128 << dev->pcie_mpss;
2676
2677                if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2678                    dev->bus->self)
2679
2680                        /*
2681                         * For "Performance", the assumption is made that
2682                         * downstream communication will never be larger than
2683                         * the MRRS.  So, the MPS only needs to be configured
2684                         * for the upstream communication.  This being the case,
2685                         * walk from the top down and set the MPS of the child
2686                         * to that of the parent bus.
2687                         *
2688                         * Configure the device MPS with the smaller of the
2689                         * device MPSS or the bridge MPS (which is assumed to be
2690                         * properly configured at this point to the largest
2691                         * allowable MPS based on its parent bus).
2692                         */
2693                        mps = min(mps, pcie_get_mps(dev->bus->self));
2694        }
2695
2696        rc = pcie_set_mps(dev, mps);
2697        if (rc)
2698                pci_err(dev, "Failed attempting to set the MPS\n");
2699}
2700
2701static void pcie_write_mrrs(struct pci_dev *dev)
2702{
2703        int rc, mrrs;
2704
2705        /*
2706         * In the "safe" case, do not configure the MRRS.  There appear to be
2707         * issues with setting MRRS to 0 on a number of devices.
2708         */
2709        if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2710                return;
2711
2712        /*
2713         * For max performance, the MRRS must be set to the largest supported
2714         * value.  However, it cannot be configured larger than the MPS the
2715         * device or the bus can support.  This should already be properly
2716         * configured by a prior call to pcie_write_mps().
2717         */
2718        mrrs = pcie_get_mps(dev);
2719
2720        /*
2721         * MRRS is a R/W register.  Invalid values can be written, but a
2722         * subsequent read will verify if the value is acceptable or not.
2723         * If the MRRS value provided is not acceptable (e.g., too large),
2724         * shrink the value until it is acceptable to the HW.
2725         */
2726        while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
2727                rc = pcie_set_readrq(dev, mrrs);
2728                if (!rc)
2729                        break;
2730
2731                pci_warn(dev, "Failed attempting to set the MRRS\n");
2732                mrrs /= 2;
2733        }
2734
2735        if (mrrs < 128)
2736                pci_err(dev, "MRRS was unable to be configured with a safe value.  If problems are experienced, try running with pci=pcie_bus_safe\n");
2737}
2738
2739static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
2740{
2741        int mps, orig_mps;
2742
2743        if (!pci_is_pcie(dev))
2744                return 0;
2745
2746        if (pcie_bus_config == PCIE_BUS_TUNE_OFF ||
2747            pcie_bus_config == PCIE_BUS_DEFAULT)
2748                return 0;
2749
2750        mps = 128 << *(u8 *)data;
2751        orig_mps = pcie_get_mps(dev);
2752
2753        pcie_write_mps(dev, mps);
2754        pcie_write_mrrs(dev);
2755
2756        pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
2757                 pcie_get_mps(dev), 128 << dev->pcie_mpss,
2758                 orig_mps, pcie_get_readrq(dev));
2759
2760        return 0;
2761}
2762
2763/*
2764 * pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
2765 * parents then children fashion.  If this changes, then this code will not
2766 * work as designed.
2767 */
2768void pcie_bus_configure_settings(struct pci_bus *bus)
2769{
2770        u8 smpss = 0;
2771
2772        if (!bus->self)
2773                return;
2774
2775        if (!pci_is_pcie(bus->self))
2776                return;
2777
2778        /*
2779         * FIXME - Peer to peer DMA is possible, though the endpoint would need
2780         * to be aware of the MPS of the destination.  To work around this,
2781         * simply force the MPS of the entire system to the smallest possible.
2782         */
2783        if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2784                smpss = 0;
2785
2786        if (pcie_bus_config == PCIE_BUS_SAFE) {
2787                smpss = bus->self->pcie_mpss;
2788
2789                pcie_find_smpss(bus->self, &smpss);
2790                pci_walk_bus(bus, pcie_find_smpss, &smpss);
2791        }
2792
2793        pcie_bus_configure_set(bus->self, &smpss);
2794        pci_walk_bus(bus, pcie_bus_configure_set, &smpss);
2795}
2796EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
2797
2798/*
2799 * Called after each bus is probed, but before its children are examined.  This
2800 * is marked as __weak because multiple architectures define it.
2801 */
2802void __weak pcibios_fixup_bus(struct pci_bus *bus)
2803{
2804       /* nothing to do, expected to be removed in the future */
2805}
2806
2807/**
2808 * pci_scan_child_bus_extend() - Scan devices below a bus
2809 * @bus: Bus to scan for devices
2810 * @available_buses: Total number of buses available (%0 does not try to
2811 *                   extend beyond the minimal)
2812 *
2813 * Scans devices below @bus including subordinate buses. Returns new
2814 * subordinate number including all the found devices. Passing
2815 * @available_buses causes the remaining bus space to be distributed
2816 * equally between hotplug-capable bridges to allow future extension of the
2817 * hierarchy.
2818 */
2819static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
2820                                              unsigned int available_buses)
2821{
2822        unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0;
2823        unsigned int start = bus->busn_res.start;
2824        unsigned int devfn, fn, cmax, max = start;
2825        struct pci_dev *dev;
2826        int nr_devs;
2827
2828        dev_dbg(&bus->dev, "scanning bus\n");
2829
2830        /* Go find them, Rover! */
2831        for (devfn = 0; devfn < 256; devfn += 8) {
2832                nr_devs = pci_scan_slot(bus, devfn);
2833
2834                /*
2835                 * The Jailhouse hypervisor may pass individual functions of a
2836                 * multi-function device to a guest without passing function 0.
2837                 * Look for them as well.
2838                 */
2839                if (jailhouse_paravirt() && nr_devs == 0) {
2840                        for (fn = 1; fn < 8; fn++) {
2841                                dev = pci_scan_single_device(bus, devfn + fn);
2842                                if (dev)
2843                                        dev->multifunction = 1;
2844                        }
2845                }
2846        }
2847
2848        /* Reserve buses for SR-IOV capability */
2849        used_buses = pci_iov_bus_range(bus);
2850        max += used_buses;
2851
2852        /*
2853         * After performing arch-dependent fixup of the bus, look behind
2854         * all PCI-to-PCI bridges on this bus.
2855         */
2856        if (!bus->is_added) {
2857                dev_dbg(&bus->dev, "fixups for bus\n");
2858                pcibios_fixup_bus(bus);
2859                bus->is_added = 1;
2860        }
2861
2862        /*
2863         * Calculate how many hotplug bridges and normal bridges there
2864         * are on this bus. We will distribute the additional available
2865         * buses between hotplug bridges.
2866         */
2867        for_each_pci_bridge(dev, bus) {
2868                if (dev->is_hotplug_bridge)
2869                        hotplug_bridges++;
2870                else
2871                        normal_bridges++;
2872        }
2873
2874        /*
2875         * Scan bridges that are already configured. We don't touch them
2876         * unless they are misconfigured (which will be done in the second
2877         * scan below).
2878         */
2879        for_each_pci_bridge(dev, bus) {
2880                cmax = max;
2881                max = pci_scan_bridge_extend(bus, dev, max, 0, 0);
2882
2883                /*
2884                 * Reserve one bus for each bridge now to avoid extending
2885                 * hotplug bridges too much during the second scan below.
2886                 */
2887                used_buses++;
2888                if (cmax - max > 1)
2889                        used_buses += cmax - max - 1;
2890        }
2891
2892        /* Scan bridges that need to be reconfigured */
2893        for_each_pci_bridge(dev, bus) {
2894                unsigned int buses = 0;
2895
2896                if (!hotplug_bridges && normal_bridges == 1) {
2897
2898                        /*
2899                         * There is only one bridge on the bus (upstream
2900                         * port) so it gets all available buses which it
2901                         * can then distribute to the possible hotplug
2902                         * bridges below.
2903                         */
2904                        buses = available_buses;
2905                } else if (dev->is_hotplug_bridge) {
2906
2907                        /*
2908                         * Distribute the extra buses between hotplug
2909                         * bridges if any.
2910                         */
2911                        buses = available_buses / hotplug_bridges;
2912                        buses = min(buses, available_buses - used_buses + 1);
2913                }
2914
2915                cmax = max;
2916                max = pci_scan_bridge_extend(bus, dev, cmax, buses, 1);
2917                /* One bus is already accounted so don't add it again */
2918                if (max - cmax > 1)
2919                        used_buses += max - cmax - 1;
2920        }
2921
2922        /*
2923         * Make sure a hotplug bridge has at least the minimum requested
2924         * number of buses but allow it to grow up to the maximum available
2925         * bus number of there is room.
2926         */
2927        if (bus->self && bus->self->is_hotplug_bridge) {
2928                used_buses = max_t(unsigned int, available_buses,
2929                                   pci_hotplug_bus_size - 1);
2930                if (max - start < used_buses) {
2931                        max = start + used_buses;
2932
2933                        /* Do not allocate more buses than we have room left */
2934                        if (max > bus->busn_res.end)
2935                                max = bus->busn_res.end;
2936
2937                        dev_dbg(&bus->dev, "%pR extended by %#02x\n",
2938                                &bus->busn_res, max - start);
2939                }
2940        }
2941
2942        /*
2943         * We've scanned the bus and so we know all about what's on
2944         * the other side of any bridges that may be on this bus plus
2945         * any devices.
2946         *
2947         * Return how far we've got finding sub-buses.
2948         */
2949        dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
2950        return max;
2951}
2952
2953/**
2954 * pci_scan_child_bus() - Scan devices below a bus
2955 * @bus: Bus to scan for devices
2956 *
2957 * Scans devices below @bus including subordinate buses. Returns new
2958 * subordinate number including all the found devices.
2959 */
2960unsigned int pci_scan_child_bus(struct pci_bus *bus)
2961{
2962        return pci_scan_child_bus_extend(bus, 0);
2963}
2964EXPORT_SYMBOL_GPL(pci_scan_child_bus);
2965
2966/**
2967 * pcibios_root_bridge_prepare - Platform-specific host bridge setup
2968 * @bridge: Host bridge to set up
2969 *
2970 * Default empty implementation.  Replace with an architecture-specific setup
2971 * routine, if necessary.
2972 */
2973int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
2974{
2975        return 0;
2976}
2977
2978void __weak pcibios_add_bus(struct pci_bus *bus)
2979{
2980}
2981
2982void __weak pcibios_remove_bus(struct pci_bus *bus)
2983{
2984}
2985
2986struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
2987                struct pci_ops *ops, void *sysdata, struct list_head *resources)
2988{
2989        int error;
2990        struct pci_host_bridge *bridge;
2991
2992        bridge = pci_alloc_host_bridge(0);
2993        if (!bridge)
2994                return NULL;
2995
2996        bridge->dev.parent = parent;
2997
2998        list_splice_init(resources, &bridge->windows);
2999        bridge->sysdata = sysdata;
3000        bridge->busnr = bus;
3001        bridge->ops = ops;
3002
3003        error = pci_register_host_bridge(bridge);
3004        if (error < 0)
3005                goto err_out;
3006
3007        return bridge->bus;
3008
3009err_out:
3010        put_device(&bridge->dev);
3011        return NULL;
3012}
3013EXPORT_SYMBOL_GPL(pci_create_root_bus);
3014
3015int pci_host_probe(struct pci_host_bridge *bridge)
3016{
3017        struct pci_bus *bus, *child;
3018        int ret;
3019
3020        ret = pci_scan_root_bus_bridge(bridge);
3021        if (ret < 0) {
3022                dev_err(bridge->dev.parent, "Scanning root bridge failed");
3023                return ret;
3024        }
3025
3026        bus = bridge->bus;
3027
3028        /*
3029         * We insert PCI resources into the iomem_resource and
3030         * ioport_resource trees in either pci_bus_claim_resources()
3031         * or pci_bus_assign_resources().
3032         */
3033        if (pci_has_flag(PCI_PROBE_ONLY)) {
3034                pci_bus_claim_resources(bus);
3035        } else {
3036                pci_bus_size_bridges(bus);
3037                pci_bus_assign_resources(bus);
3038
3039                list_for_each_entry(child, &bus->children, node)
3040                        pcie_bus_configure_settings(child);
3041        }
3042
3043        pci_bus_add_devices(bus);
3044        return 0;
3045}
3046EXPORT_SYMBOL_GPL(pci_host_probe);
3047
3048int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
3049{
3050        struct resource *res = &b->busn_res;
3051        struct resource *parent_res, *conflict;
3052
3053        res->start = bus;
3054        res->end = bus_max;
3055        res->flags = IORESOURCE_BUS;
3056
3057        if (!pci_is_root_bus(b))
3058                parent_res = &b->parent->busn_res;
3059        else {
3060                parent_res = get_pci_domain_busn_res(pci_domain_nr(b));
3061                res->flags |= IORESOURCE_PCI_FIXED;
3062        }
3063
3064        conflict = request_resource_conflict(parent_res, res);
3065
3066        if (conflict)
3067                dev_info(&b->dev,
3068                           "busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3069                            res, pci_is_root_bus(b) ? "domain " : "",
3070                            parent_res, conflict->name, conflict);
3071
3072        return conflict == NULL;
3073}
3074
3075int pci_bus_update_busn_res_end(struct pci_bus *b, int bus_max)
3076{
3077        struct resource *res = &b->busn_res;
3078        struct resource old_res = *res;
3079        resource_size_t size;
3080        int ret;
3081
3082        if (res->start > bus_max)
3083                return -EINVAL;
3084
3085        size = bus_max - res->start + 1;
3086        ret = adjust_resource(res, res->start, size);
3087        dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3088                        &old_res, ret ? "can not be" : "is", bus_max);
3089
3090        if (!ret && !res->parent)
3091                pci_bus_insert_busn_res(b, res->start, res->end);
3092
3093        return ret;
3094}
3095
3096void pci_bus_release_busn_res(struct pci_bus *b)
3097{
3098        struct resource *res = &b->busn_res;
3099        int ret;
3100
3101        if (!res->flags || !res->parent)
3102                return;
3103
3104        ret = release_resource(res);
3105        dev_info(&b->dev, "busn_res: %pR %s released\n",
3106                        res, ret ? "can not be" : "is");
3107}
3108
3109int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3110{
3111        struct resource_entry *window;
3112        bool found = false;
3113        struct pci_bus *b;
3114        int max, bus, ret;
3115
3116        if (!bridge)
3117                return -EINVAL;
3118
3119        resource_list_for_each_entry(window, &bridge->windows)
3120                if (window->res->flags & IORESOURCE_BUS) {
3121                        bridge->busnr = window->res->start;
3122                        found = true;
3123                        break;
3124                }
3125
3126        ret = pci_register_host_bridge(bridge);
3127        if (ret < 0)
3128                return ret;
3129
3130        b = bridge->bus;
3131        bus = bridge->busnr;
3132
3133        if (!found) {
3134                dev_info(&b->dev,
3135                 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3136                        bus);
3137                pci_bus_insert_busn_res(b, bus, 255);
3138        }
3139
3140        max = pci_scan_child_bus(b);
3141
3142        if (!found)
3143                pci_bus_update_busn_res_end(b, max);
3144
3145        return 0;
3146}
3147EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3148
3149struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
3150                struct pci_ops *ops, void *sysdata, struct list_head *resources)
3151{
3152        struct resource_entry *window;
3153        bool found = false;
3154        struct pci_bus *b;
3155        int max;
3156
3157        resource_list_for_each_entry(window, resources)
3158                if (window->res->flags & IORESOURCE_BUS) {
3159                        found = true;
3160                        break;
3161                }
3162
3163        b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3164        if (!b)
3165                return NULL;
3166
3167        if (!found) {
3168                dev_info(&b->dev,
3169                 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3170                        bus);
3171                pci_bus_insert_busn_res(b, bus, 255);
3172        }
3173
3174        max = pci_scan_child_bus(b);
3175
3176        if (!found)
3177                pci_bus_update_busn_res_end(b, max);
3178
3179        return b;
3180}
3181EXPORT_SYMBOL(pci_scan_root_bus);
3182
3183struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops,
3184                                        void *sysdata)
3185{
3186        LIST_HEAD(resources);
3187        struct pci_bus *b;
3188
3189        pci_add_resource(&resources, &ioport_resource);
3190        pci_add_resource(&resources, &iomem_resource);
3191        pci_add_resource(&resources, &busn_resource);
3192        b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3193        if (b) {
3194                pci_scan_child_bus(b);
3195        } else {
3196                pci_free_resource_list(&resources);
3197        }
3198        return b;
3199}
3200EXPORT_SYMBOL(pci_scan_bus);
3201
3202/**
3203 * pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3204 * @bridge: PCI bridge for the bus to scan
3205 *
3206 * Scan a PCI bus and child buses for new devices, add them,
3207 * and enable them, resizing bridge mmio/io resource if necessary
3208 * and possible.  The caller must ensure the child devices are already
3209 * removed for resizing to occur.
3210 *
3211 * Returns the max number of subordinate bus discovered.
3212 */
3213unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3214{
3215        unsigned int max;
3216        struct pci_bus *bus = bridge->subordinate;
3217
3218        max = pci_scan_child_bus(bus);
3219
3220        pci_assign_unassigned_bridge_resources(bridge);
3221
3222        pci_bus_add_devices(bus);
3223
3224        return max;
3225}
3226
3227/**
3228 * pci_rescan_bus - Scan a PCI bus for devices
3229 * @bus: PCI bus to scan
3230 *
3231 * Scan a PCI bus and child buses for new devices, add them,
3232 * and enable them.
3233 *
3234 * Returns the max number of subordinate bus discovered.
3235 */
3236unsigned int pci_rescan_bus(struct pci_bus *bus)
3237{
3238        unsigned int max;
3239
3240        max = pci_scan_child_bus(bus);
3241        pci_assign_unassigned_bus_resources(bus);
3242        pci_bus_add_devices(bus);
3243
3244        return max;
3245}
3246EXPORT_SYMBOL_GPL(pci_rescan_bus);
3247
3248/*
3249 * pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3250 * routines should always be executed under this mutex.
3251 */
3252static DEFINE_MUTEX(pci_rescan_remove_lock);
3253
3254void pci_lock_rescan_remove(void)
3255{
3256        mutex_lock(&pci_rescan_remove_lock);
3257}
3258EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3259
3260void pci_unlock_rescan_remove(void)
3261{
3262        mutex_unlock(&pci_rescan_remove_lock);
3263}
3264EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3265
3266static int __init pci_sort_bf_cmp(const struct device *d_a,
3267                                  const struct device *d_b)
3268{
3269        const struct pci_dev *a = to_pci_dev(d_a);
3270        const struct pci_dev *b = to_pci_dev(d_b);
3271
3272        if      (pci_domain_nr(a->bus) < pci_domain_nr(b->bus)) return -1;
3273        else if (pci_domain_nr(a->bus) > pci_domain_nr(b->bus)) return  1;
3274
3275        if      (a->bus->number < b->bus->number) return -1;
3276        else if (a->bus->number > b->bus->number) return  1;
3277
3278        if      (a->devfn < b->devfn) return -1;
3279        else if (a->devfn > b->devfn) return  1;
3280
3281        return 0;
3282}
3283
3284void __init pci_sort_breadthfirst(void)
3285{
3286        bus_sort_breadthfirst(&pci_bus_type, &pci_sort_bf_cmp);
3287}
3288
3289int pci_hp_add_bridge(struct pci_dev *dev)
3290{
3291        struct pci_bus *parent = dev->bus;
3292        int busnr, start = parent->busn_res.start;
3293        unsigned int available_buses = 0;
3294        int end = parent->busn_res.end;
3295
3296        for (busnr = start; busnr <= end; busnr++) {
3297                if (!pci_find_bus(pci_domain_nr(parent), busnr))
3298                        break;
3299        }
3300        if (busnr-- > end) {
3301                pci_err(dev, "No bus number available for hot-added bridge\n");
3302                return -1;
3303        }
3304
3305        /* Scan bridges that are already configured */
3306        busnr = pci_scan_bridge(parent, dev, busnr, 0);
3307
3308        /*
3309         * Distribute the available bus numbers between hotplug-capable
3310         * bridges to make extending the chain later possible.
3311         */
3312        available_buses = end - busnr;
3313
3314        /* Scan bridges that need to be reconfigured */
3315        pci_scan_bridge_extend(parent, dev, busnr, available_buses, 1);
3316
3317        if (!dev->subordinate)
3318                return -1;
3319
3320        return 0;
3321}
3322EXPORT_SYMBOL_GPL(pci_hp_add_bridge);
3323