linux/drivers/mmc/host/sdhci.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
   4 *
   5 *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
   6 *
   7 * Thanks to the following companies for their support:
   8 *
   9 *     - JMicron (hardware and technical support)
  10 */
  11
  12#include <linux/bitfield.h>
  13#include <linux/delay.h>
  14#include <linux/dmaengine.h>
  15#include <linux/ktime.h>
  16#include <linux/highmem.h>
  17#include <linux/io.h>
  18#include <linux/module.h>
  19#include <linux/dma-mapping.h>
  20#include <linux/slab.h>
  21#include <linux/scatterlist.h>
  22#include <linux/sizes.h>
  23#include <linux/regulator/consumer.h>
  24#include <linux/pm_runtime.h>
  25#include <linux/of.h>
  26
  27#include <linux/leds.h>
  28
  29#include <linux/mmc/mmc.h>
  30#include <linux/mmc/host.h>
  31#include <linux/mmc/card.h>
  32#include <linux/mmc/sdio.h>
  33#include <linux/mmc/slot-gpio.h>
  34
  35#include "sdhci.h"
  36
  37#define DRIVER_NAME "sdhci"
  38
  39#define DBG(f, x...) \
  40        pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  41
  42#define SDHCI_DUMP(f, x...) \
  43        pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  44
  45#define MAX_TUNING_LOOP 40
  46
  47static unsigned int debug_quirks = 0;
  48static unsigned int debug_quirks2;
  49
  50static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
  51
  52static bool sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd);
  53
  54void sdhci_dumpregs(struct sdhci_host *host)
  55{
  56        SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
  57
  58        SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
  59                   sdhci_readl(host, SDHCI_DMA_ADDRESS),
  60                   sdhci_readw(host, SDHCI_HOST_VERSION));
  61        SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
  62                   sdhci_readw(host, SDHCI_BLOCK_SIZE),
  63                   sdhci_readw(host, SDHCI_BLOCK_COUNT));
  64        SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
  65                   sdhci_readl(host, SDHCI_ARGUMENT),
  66                   sdhci_readw(host, SDHCI_TRANSFER_MODE));
  67        SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
  68                   sdhci_readl(host, SDHCI_PRESENT_STATE),
  69                   sdhci_readb(host, SDHCI_HOST_CONTROL));
  70        SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
  71                   sdhci_readb(host, SDHCI_POWER_CONTROL),
  72                   sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
  73        SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
  74                   sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
  75                   sdhci_readw(host, SDHCI_CLOCK_CONTROL));
  76        SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
  77                   sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
  78                   sdhci_readl(host, SDHCI_INT_STATUS));
  79        SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
  80                   sdhci_readl(host, SDHCI_INT_ENABLE),
  81                   sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
  82        SDHCI_DUMP("ACmd stat: 0x%08x | Slot int: 0x%08x\n",
  83                   sdhci_readw(host, SDHCI_AUTO_CMD_STATUS),
  84                   sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
  85        SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
  86                   sdhci_readl(host, SDHCI_CAPABILITIES),
  87                   sdhci_readl(host, SDHCI_CAPABILITIES_1));
  88        SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
  89                   sdhci_readw(host, SDHCI_COMMAND),
  90                   sdhci_readl(host, SDHCI_MAX_CURRENT));
  91        SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
  92                   sdhci_readl(host, SDHCI_RESPONSE),
  93                   sdhci_readl(host, SDHCI_RESPONSE + 4));
  94        SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
  95                   sdhci_readl(host, SDHCI_RESPONSE + 8),
  96                   sdhci_readl(host, SDHCI_RESPONSE + 12));
  97        SDHCI_DUMP("Host ctl2: 0x%08x\n",
  98                   sdhci_readw(host, SDHCI_HOST_CONTROL2));
  99
 100        if (host->flags & SDHCI_USE_ADMA) {
 101                if (host->flags & SDHCI_USE_64_BIT_DMA) {
 102                        SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
 103                                   sdhci_readl(host, SDHCI_ADMA_ERROR),
 104                                   sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
 105                                   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 106                } else {
 107                        SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
 108                                   sdhci_readl(host, SDHCI_ADMA_ERROR),
 109                                   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 110                }
 111        }
 112
 113        if (host->ops->dump_vendor_regs)
 114                host->ops->dump_vendor_regs(host);
 115
 116        SDHCI_DUMP("============================================\n");
 117}
 118EXPORT_SYMBOL_GPL(sdhci_dumpregs);
 119
 120/*****************************************************************************\
 121 *                                                                           *
 122 * Low level functions                                                       *
 123 *                                                                           *
 124\*****************************************************************************/
 125
 126static void sdhci_do_enable_v4_mode(struct sdhci_host *host)
 127{
 128        u16 ctrl2;
 129
 130        ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
 131        if (ctrl2 & SDHCI_CTRL_V4_MODE)
 132                return;
 133
 134        ctrl2 |= SDHCI_CTRL_V4_MODE;
 135        sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
 136}
 137
 138/*
 139 * This can be called before sdhci_add_host() by Vendor's host controller
 140 * driver to enable v4 mode if supported.
 141 */
 142void sdhci_enable_v4_mode(struct sdhci_host *host)
 143{
 144        host->v4_mode = true;
 145        sdhci_do_enable_v4_mode(host);
 146}
 147EXPORT_SYMBOL_GPL(sdhci_enable_v4_mode);
 148
 149static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
 150{
 151        return cmd->data || cmd->flags & MMC_RSP_BUSY;
 152}
 153
 154static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
 155{
 156        u32 present;
 157
 158        if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
 159            !mmc_card_is_removable(host->mmc) || mmc_can_gpio_cd(host->mmc))
 160                return;
 161
 162        if (enable) {
 163                present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
 164                                      SDHCI_CARD_PRESENT;
 165
 166                host->ier |= present ? SDHCI_INT_CARD_REMOVE :
 167                                       SDHCI_INT_CARD_INSERT;
 168        } else {
 169                host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 170        }
 171
 172        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 173        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 174}
 175
 176static void sdhci_enable_card_detection(struct sdhci_host *host)
 177{
 178        sdhci_set_card_detection(host, true);
 179}
 180
 181static void sdhci_disable_card_detection(struct sdhci_host *host)
 182{
 183        sdhci_set_card_detection(host, false);
 184}
 185
 186static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
 187{
 188        if (host->bus_on)
 189                return;
 190        host->bus_on = true;
 191        pm_runtime_get_noresume(mmc_dev(host->mmc));
 192}
 193
 194static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
 195{
 196        if (!host->bus_on)
 197                return;
 198        host->bus_on = false;
 199        pm_runtime_put_noidle(mmc_dev(host->mmc));
 200}
 201
 202void sdhci_reset(struct sdhci_host *host, u8 mask)
 203{
 204        ktime_t timeout;
 205
 206        sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
 207
 208        if (mask & SDHCI_RESET_ALL) {
 209                host->clock = 0;
 210                /* Reset-all turns off SD Bus Power */
 211                if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
 212                        sdhci_runtime_pm_bus_off(host);
 213        }
 214
 215        /* Wait max 100 ms */
 216        timeout = ktime_add_ms(ktime_get(), 100);
 217
 218        /* hw clears the bit when it's done */
 219        while (1) {
 220                bool timedout = ktime_after(ktime_get(), timeout);
 221
 222                if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask))
 223                        break;
 224                if (timedout) {
 225                        pr_err("%s: Reset 0x%x never completed.\n",
 226                                mmc_hostname(host->mmc), (int)mask);
 227                        sdhci_dumpregs(host);
 228                        return;
 229                }
 230                udelay(10);
 231        }
 232}
 233EXPORT_SYMBOL_GPL(sdhci_reset);
 234
 235static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
 236{
 237        if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 238                struct mmc_host *mmc = host->mmc;
 239
 240                if (!mmc->ops->get_cd(mmc))
 241                        return;
 242        }
 243
 244        host->ops->reset(host, mask);
 245
 246        if (mask & SDHCI_RESET_ALL) {
 247                if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 248                        if (host->ops->enable_dma)
 249                                host->ops->enable_dma(host);
 250                }
 251
 252                /* Resetting the controller clears many */
 253                host->preset_enabled = false;
 254        }
 255}
 256
 257static void sdhci_set_default_irqs(struct sdhci_host *host)
 258{
 259        host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
 260                    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
 261                    SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
 262                    SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
 263                    SDHCI_INT_RESPONSE;
 264
 265        if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
 266            host->tuning_mode == SDHCI_TUNING_MODE_3)
 267                host->ier |= SDHCI_INT_RETUNE;
 268
 269        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 270        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 271}
 272
 273static void sdhci_config_dma(struct sdhci_host *host)
 274{
 275        u8 ctrl;
 276        u16 ctrl2;
 277
 278        if (host->version < SDHCI_SPEC_200)
 279                return;
 280
 281        ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 282
 283        /*
 284         * Always adjust the DMA selection as some controllers
 285         * (e.g. JMicron) can't do PIO properly when the selection
 286         * is ADMA.
 287         */
 288        ctrl &= ~SDHCI_CTRL_DMA_MASK;
 289        if (!(host->flags & SDHCI_REQ_USE_DMA))
 290                goto out;
 291
 292        /* Note if DMA Select is zero then SDMA is selected */
 293        if (host->flags & SDHCI_USE_ADMA)
 294                ctrl |= SDHCI_CTRL_ADMA32;
 295
 296        if (host->flags & SDHCI_USE_64_BIT_DMA) {
 297                /*
 298                 * If v4 mode, all supported DMA can be 64-bit addressing if
 299                 * controller supports 64-bit system address, otherwise only
 300                 * ADMA can support 64-bit addressing.
 301                 */
 302                if (host->v4_mode) {
 303                        ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
 304                        ctrl2 |= SDHCI_CTRL_64BIT_ADDR;
 305                        sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
 306                } else if (host->flags & SDHCI_USE_ADMA) {
 307                        /*
 308                         * Don't need to undo SDHCI_CTRL_ADMA32 in order to
 309                         * set SDHCI_CTRL_ADMA64.
 310                         */
 311                        ctrl |= SDHCI_CTRL_ADMA64;
 312                }
 313        }
 314
 315out:
 316        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 317}
 318
 319static void sdhci_init(struct sdhci_host *host, int soft)
 320{
 321        struct mmc_host *mmc = host->mmc;
 322        unsigned long flags;
 323
 324        if (soft)
 325                sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 326        else
 327                sdhci_do_reset(host, SDHCI_RESET_ALL);
 328
 329        if (host->v4_mode)
 330                sdhci_do_enable_v4_mode(host);
 331
 332        spin_lock_irqsave(&host->lock, flags);
 333        sdhci_set_default_irqs(host);
 334        spin_unlock_irqrestore(&host->lock, flags);
 335
 336        host->cqe_on = false;
 337
 338        if (soft) {
 339                /* force clock reconfiguration */
 340                host->clock = 0;
 341                mmc->ops->set_ios(mmc, &mmc->ios);
 342        }
 343}
 344
 345static void sdhci_reinit(struct sdhci_host *host)
 346{
 347        u32 cd = host->ier & (SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 348
 349        sdhci_init(host, 0);
 350        sdhci_enable_card_detection(host);
 351
 352        /*
 353         * A change to the card detect bits indicates a change in present state,
 354         * refer sdhci_set_card_detection(). A card detect interrupt might have
 355         * been missed while the host controller was being reset, so trigger a
 356         * rescan to check.
 357         */
 358        if (cd != (host->ier & (SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT)))
 359                mmc_detect_change(host->mmc, msecs_to_jiffies(200));
 360}
 361
 362static void __sdhci_led_activate(struct sdhci_host *host)
 363{
 364        u8 ctrl;
 365
 366        if (host->quirks & SDHCI_QUIRK_NO_LED)
 367                return;
 368
 369        ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 370        ctrl |= SDHCI_CTRL_LED;
 371        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 372}
 373
 374static void __sdhci_led_deactivate(struct sdhci_host *host)
 375{
 376        u8 ctrl;
 377
 378        if (host->quirks & SDHCI_QUIRK_NO_LED)
 379                return;
 380
 381        ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 382        ctrl &= ~SDHCI_CTRL_LED;
 383        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 384}
 385
 386#if IS_REACHABLE(CONFIG_LEDS_CLASS)
 387static void sdhci_led_control(struct led_classdev *led,
 388                              enum led_brightness brightness)
 389{
 390        struct sdhci_host *host = container_of(led, struct sdhci_host, led);
 391        unsigned long flags;
 392
 393        spin_lock_irqsave(&host->lock, flags);
 394
 395        if (host->runtime_suspended)
 396                goto out;
 397
 398        if (brightness == LED_OFF)
 399                __sdhci_led_deactivate(host);
 400        else
 401                __sdhci_led_activate(host);
 402out:
 403        spin_unlock_irqrestore(&host->lock, flags);
 404}
 405
 406static int sdhci_led_register(struct sdhci_host *host)
 407{
 408        struct mmc_host *mmc = host->mmc;
 409
 410        if (host->quirks & SDHCI_QUIRK_NO_LED)
 411                return 0;
 412
 413        snprintf(host->led_name, sizeof(host->led_name),
 414                 "%s::", mmc_hostname(mmc));
 415
 416        host->led.name = host->led_name;
 417        host->led.brightness = LED_OFF;
 418        host->led.default_trigger = mmc_hostname(mmc);
 419        host->led.brightness_set = sdhci_led_control;
 420
 421        return led_classdev_register(mmc_dev(mmc), &host->led);
 422}
 423
 424static void sdhci_led_unregister(struct sdhci_host *host)
 425{
 426        if (host->quirks & SDHCI_QUIRK_NO_LED)
 427                return;
 428
 429        led_classdev_unregister(&host->led);
 430}
 431
 432static inline void sdhci_led_activate(struct sdhci_host *host)
 433{
 434}
 435
 436static inline void sdhci_led_deactivate(struct sdhci_host *host)
 437{
 438}
 439
 440#else
 441
 442static inline int sdhci_led_register(struct sdhci_host *host)
 443{
 444        return 0;
 445}
 446
 447static inline void sdhci_led_unregister(struct sdhci_host *host)
 448{
 449}
 450
 451static inline void sdhci_led_activate(struct sdhci_host *host)
 452{
 453        __sdhci_led_activate(host);
 454}
 455
 456static inline void sdhci_led_deactivate(struct sdhci_host *host)
 457{
 458        __sdhci_led_deactivate(host);
 459}
 460
 461#endif
 462
 463static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
 464                            unsigned long timeout)
 465{
 466        if (sdhci_data_line_cmd(mrq->cmd))
 467                mod_timer(&host->data_timer, timeout);
 468        else
 469                mod_timer(&host->timer, timeout);
 470}
 471
 472static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
 473{
 474        if (sdhci_data_line_cmd(mrq->cmd))
 475                del_timer(&host->data_timer);
 476        else
 477                del_timer(&host->timer);
 478}
 479
 480static inline bool sdhci_has_requests(struct sdhci_host *host)
 481{
 482        return host->cmd || host->data_cmd;
 483}
 484
 485/*****************************************************************************\
 486 *                                                                           *
 487 * Core functions                                                            *
 488 *                                                                           *
 489\*****************************************************************************/
 490
 491static void sdhci_read_block_pio(struct sdhci_host *host)
 492{
 493        unsigned long flags;
 494        size_t blksize, len, chunk;
 495        u32 scratch;
 496        u8 *buf;
 497
 498        DBG("PIO reading\n");
 499
 500        blksize = host->data->blksz;
 501        chunk = 0;
 502
 503        local_irq_save(flags);
 504
 505        while (blksize) {
 506                BUG_ON(!sg_miter_next(&host->sg_miter));
 507
 508                len = min(host->sg_miter.length, blksize);
 509
 510                blksize -= len;
 511                host->sg_miter.consumed = len;
 512
 513                buf = host->sg_miter.addr;
 514
 515                while (len) {
 516                        if (chunk == 0) {
 517                                scratch = sdhci_readl(host, SDHCI_BUFFER);
 518                                chunk = 4;
 519                        }
 520
 521                        *buf = scratch & 0xFF;
 522
 523                        buf++;
 524                        scratch >>= 8;
 525                        chunk--;
 526                        len--;
 527                }
 528        }
 529
 530        sg_miter_stop(&host->sg_miter);
 531
 532        local_irq_restore(flags);
 533}
 534
 535static void sdhci_write_block_pio(struct sdhci_host *host)
 536{
 537        unsigned long flags;
 538        size_t blksize, len, chunk;
 539        u32 scratch;
 540        u8 *buf;
 541
 542        DBG("PIO writing\n");
 543
 544        blksize = host->data->blksz;
 545        chunk = 0;
 546        scratch = 0;
 547
 548        local_irq_save(flags);
 549
 550        while (blksize) {
 551                BUG_ON(!sg_miter_next(&host->sg_miter));
 552
 553                len = min(host->sg_miter.length, blksize);
 554
 555                blksize -= len;
 556                host->sg_miter.consumed = len;
 557
 558                buf = host->sg_miter.addr;
 559
 560                while (len) {
 561                        scratch |= (u32)*buf << (chunk * 8);
 562
 563                        buf++;
 564                        chunk++;
 565                        len--;
 566
 567                        if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
 568                                sdhci_writel(host, scratch, SDHCI_BUFFER);
 569                                chunk = 0;
 570                                scratch = 0;
 571                        }
 572                }
 573        }
 574
 575        sg_miter_stop(&host->sg_miter);
 576
 577        local_irq_restore(flags);
 578}
 579
 580static void sdhci_transfer_pio(struct sdhci_host *host)
 581{
 582        u32 mask;
 583
 584        if (host->blocks == 0)
 585                return;
 586
 587        if (host->data->flags & MMC_DATA_READ)
 588                mask = SDHCI_DATA_AVAILABLE;
 589        else
 590                mask = SDHCI_SPACE_AVAILABLE;
 591
 592        /*
 593         * Some controllers (JMicron JMB38x) mess up the buffer bits
 594         * for transfers < 4 bytes. As long as it is just one block,
 595         * we can ignore the bits.
 596         */
 597        if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
 598                (host->data->blocks == 1))
 599                mask = ~0;
 600
 601        while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
 602                if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
 603                        udelay(100);
 604
 605                if (host->data->flags & MMC_DATA_READ)
 606                        sdhci_read_block_pio(host);
 607                else
 608                        sdhci_write_block_pio(host);
 609
 610                host->blocks--;
 611                if (host->blocks == 0)
 612                        break;
 613        }
 614
 615        DBG("PIO transfer complete.\n");
 616}
 617
 618static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 619                                  struct mmc_data *data, int cookie)
 620{
 621        int sg_count;
 622
 623        /*
 624         * If the data buffers are already mapped, return the previous
 625         * dma_map_sg() result.
 626         */
 627        if (data->host_cookie == COOKIE_PRE_MAPPED)
 628                return data->sg_count;
 629
 630        /* Bounce write requests to the bounce buffer */
 631        if (host->bounce_buffer) {
 632                unsigned int length = data->blksz * data->blocks;
 633
 634                if (length > host->bounce_buffer_size) {
 635                        pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
 636                               mmc_hostname(host->mmc), length,
 637                               host->bounce_buffer_size);
 638                        return -EIO;
 639                }
 640                if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
 641                        /* Copy the data to the bounce buffer */
 642                        if (host->ops->copy_to_bounce_buffer) {
 643                                host->ops->copy_to_bounce_buffer(host,
 644                                                                 data, length);
 645                        } else {
 646                                sg_copy_to_buffer(data->sg, data->sg_len,
 647                                                  host->bounce_buffer, length);
 648                        }
 649                }
 650                /* Switch ownership to the DMA */
 651                dma_sync_single_for_device(mmc_dev(host->mmc),
 652                                           host->bounce_addr,
 653                                           host->bounce_buffer_size,
 654                                           mmc_get_dma_dir(data));
 655                /* Just a dummy value */
 656                sg_count = 1;
 657        } else {
 658                /* Just access the data directly from memory */
 659                sg_count = dma_map_sg(mmc_dev(host->mmc),
 660                                      data->sg, data->sg_len,
 661                                      mmc_get_dma_dir(data));
 662        }
 663
 664        if (sg_count == 0)
 665                return -ENOSPC;
 666
 667        data->sg_count = sg_count;
 668        data->host_cookie = cookie;
 669
 670        return sg_count;
 671}
 672
 673static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
 674{
 675        local_irq_save(*flags);
 676        return kmap_atomic(sg_page(sg)) + sg->offset;
 677}
 678
 679static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
 680{
 681        kunmap_atomic(buffer);
 682        local_irq_restore(*flags);
 683}
 684
 685void sdhci_adma_write_desc(struct sdhci_host *host, void **desc,
 686                           dma_addr_t addr, int len, unsigned int cmd)
 687{
 688        struct sdhci_adma2_64_desc *dma_desc = *desc;
 689
 690        /* 32-bit and 64-bit descriptors have these members in same position */
 691        dma_desc->cmd = cpu_to_le16(cmd);
 692        dma_desc->len = cpu_to_le16(len);
 693        dma_desc->addr_lo = cpu_to_le32(lower_32_bits(addr));
 694
 695        if (host->flags & SDHCI_USE_64_BIT_DMA)
 696                dma_desc->addr_hi = cpu_to_le32(upper_32_bits(addr));
 697
 698        *desc += host->desc_sz;
 699}
 700EXPORT_SYMBOL_GPL(sdhci_adma_write_desc);
 701
 702static inline void __sdhci_adma_write_desc(struct sdhci_host *host,
 703                                           void **desc, dma_addr_t addr,
 704                                           int len, unsigned int cmd)
 705{
 706        if (host->ops->adma_write_desc)
 707                host->ops->adma_write_desc(host, desc, addr, len, cmd);
 708        else
 709                sdhci_adma_write_desc(host, desc, addr, len, cmd);
 710}
 711
 712static void sdhci_adma_mark_end(void *desc)
 713{
 714        struct sdhci_adma2_64_desc *dma_desc = desc;
 715
 716        /* 32-bit and 64-bit descriptors have 'cmd' in same position */
 717        dma_desc->cmd |= cpu_to_le16(ADMA2_END);
 718}
 719
 720static void sdhci_adma_table_pre(struct sdhci_host *host,
 721        struct mmc_data *data, int sg_count)
 722{
 723        struct scatterlist *sg;
 724        unsigned long flags;
 725        dma_addr_t addr, align_addr;
 726        void *desc, *align;
 727        char *buffer;
 728        int len, offset, i;
 729
 730        /*
 731         * The spec does not specify endianness of descriptor table.
 732         * We currently guess that it is LE.
 733         */
 734
 735        host->sg_count = sg_count;
 736
 737        desc = host->adma_table;
 738        align = host->align_buffer;
 739
 740        align_addr = host->align_addr;
 741
 742        for_each_sg(data->sg, sg, host->sg_count, i) {
 743                addr = sg_dma_address(sg);
 744                len = sg_dma_len(sg);
 745
 746                /*
 747                 * The SDHCI specification states that ADMA addresses must
 748                 * be 32-bit aligned. If they aren't, then we use a bounce
 749                 * buffer for the (up to three) bytes that screw up the
 750                 * alignment.
 751                 */
 752                offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
 753                         SDHCI_ADMA2_MASK;
 754                if (offset) {
 755                        if (data->flags & MMC_DATA_WRITE) {
 756                                buffer = sdhci_kmap_atomic(sg, &flags);
 757                                memcpy(align, buffer, offset);
 758                                sdhci_kunmap_atomic(buffer, &flags);
 759                        }
 760
 761                        /* tran, valid */
 762                        __sdhci_adma_write_desc(host, &desc, align_addr,
 763                                                offset, ADMA2_TRAN_VALID);
 764
 765                        BUG_ON(offset > 65536);
 766
 767                        align += SDHCI_ADMA2_ALIGN;
 768                        align_addr += SDHCI_ADMA2_ALIGN;
 769
 770                        addr += offset;
 771                        len -= offset;
 772                }
 773
 774                BUG_ON(len > 65536);
 775
 776                /* tran, valid */
 777                if (len)
 778                        __sdhci_adma_write_desc(host, &desc, addr, len,
 779                                                ADMA2_TRAN_VALID);
 780
 781                /*
 782                 * If this triggers then we have a calculation bug
 783                 * somewhere. :/
 784                 */
 785                WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
 786        }
 787
 788        if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
 789                /* Mark the last descriptor as the terminating descriptor */
 790                if (desc != host->adma_table) {
 791                        desc -= host->desc_sz;
 792                        sdhci_adma_mark_end(desc);
 793                }
 794        } else {
 795                /* Add a terminating entry - nop, end, valid */
 796                __sdhci_adma_write_desc(host, &desc, 0, 0, ADMA2_NOP_END_VALID);
 797        }
 798}
 799
 800static void sdhci_adma_table_post(struct sdhci_host *host,
 801        struct mmc_data *data)
 802{
 803        struct scatterlist *sg;
 804        int i, size;
 805        void *align;
 806        char *buffer;
 807        unsigned long flags;
 808
 809        if (data->flags & MMC_DATA_READ) {
 810                bool has_unaligned = false;
 811
 812                /* Do a quick scan of the SG list for any unaligned mappings */
 813                for_each_sg(data->sg, sg, host->sg_count, i)
 814                        if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 815                                has_unaligned = true;
 816                                break;
 817                        }
 818
 819                if (has_unaligned) {
 820                        dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
 821                                            data->sg_len, DMA_FROM_DEVICE);
 822
 823                        align = host->align_buffer;
 824
 825                        for_each_sg(data->sg, sg, host->sg_count, i) {
 826                                if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 827                                        size = SDHCI_ADMA2_ALIGN -
 828                                               (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
 829
 830                                        buffer = sdhci_kmap_atomic(sg, &flags);
 831                                        memcpy(buffer, align, size);
 832                                        sdhci_kunmap_atomic(buffer, &flags);
 833
 834                                        align += SDHCI_ADMA2_ALIGN;
 835                                }
 836                        }
 837                }
 838        }
 839}
 840
 841static void sdhci_set_adma_addr(struct sdhci_host *host, dma_addr_t addr)
 842{
 843        sdhci_writel(host, lower_32_bits(addr), SDHCI_ADMA_ADDRESS);
 844        if (host->flags & SDHCI_USE_64_BIT_DMA)
 845                sdhci_writel(host, upper_32_bits(addr), SDHCI_ADMA_ADDRESS_HI);
 846}
 847
 848static dma_addr_t sdhci_sdma_address(struct sdhci_host *host)
 849{
 850        if (host->bounce_buffer)
 851                return host->bounce_addr;
 852        else
 853                return sg_dma_address(host->data->sg);
 854}
 855
 856static void sdhci_set_sdma_addr(struct sdhci_host *host, dma_addr_t addr)
 857{
 858        if (host->v4_mode)
 859                sdhci_set_adma_addr(host, addr);
 860        else
 861                sdhci_writel(host, addr, SDHCI_DMA_ADDRESS);
 862}
 863
 864static unsigned int sdhci_target_timeout(struct sdhci_host *host,
 865                                         struct mmc_command *cmd,
 866                                         struct mmc_data *data)
 867{
 868        unsigned int target_timeout;
 869
 870        /* timeout in us */
 871        if (!data) {
 872                target_timeout = cmd->busy_timeout * 1000;
 873        } else {
 874                target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 875                if (host->clock && data->timeout_clks) {
 876                        unsigned long long val;
 877
 878                        /*
 879                         * data->timeout_clks is in units of clock cycles.
 880                         * host->clock is in Hz.  target_timeout is in us.
 881                         * Hence, us = 1000000 * cycles / Hz.  Round up.
 882                         */
 883                        val = 1000000ULL * data->timeout_clks;
 884                        if (do_div(val, host->clock))
 885                                target_timeout++;
 886                        target_timeout += val;
 887                }
 888        }
 889
 890        return target_timeout;
 891}
 892
 893static void sdhci_calc_sw_timeout(struct sdhci_host *host,
 894                                  struct mmc_command *cmd)
 895{
 896        struct mmc_data *data = cmd->data;
 897        struct mmc_host *mmc = host->mmc;
 898        struct mmc_ios *ios = &mmc->ios;
 899        unsigned char bus_width = 1 << ios->bus_width;
 900        unsigned int blksz;
 901        unsigned int freq;
 902        u64 target_timeout;
 903        u64 transfer_time;
 904
 905        target_timeout = sdhci_target_timeout(host, cmd, data);
 906        target_timeout *= NSEC_PER_USEC;
 907
 908        if (data) {
 909                blksz = data->blksz;
 910                freq = mmc->actual_clock ? : host->clock;
 911                transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width);
 912                do_div(transfer_time, freq);
 913                /* multiply by '2' to account for any unknowns */
 914                transfer_time = transfer_time * 2;
 915                /* calculate timeout for the entire data */
 916                host->data_timeout = data->blocks * target_timeout +
 917                                     transfer_time;
 918        } else {
 919                host->data_timeout = target_timeout;
 920        }
 921
 922        if (host->data_timeout)
 923                host->data_timeout += MMC_CMD_TRANSFER_TIME;
 924}
 925
 926static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,
 927                             bool *too_big)
 928{
 929        u8 count;
 930        struct mmc_data *data;
 931        unsigned target_timeout, current_timeout;
 932
 933        *too_big = true;
 934
 935        /*
 936         * If the host controller provides us with an incorrect timeout
 937         * value, just skip the check and use 0xE.  The hardware may take
 938         * longer to time out, but that's much better than having a too-short
 939         * timeout value.
 940         */
 941        if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
 942                return 0xE;
 943
 944        /* Unspecified command, asume max */
 945        if (cmd == NULL)
 946                return 0xE;
 947
 948        data = cmd->data;
 949        /* Unspecified timeout, assume max */
 950        if (!data && !cmd->busy_timeout)
 951                return 0xE;
 952
 953        /* timeout in us */
 954        target_timeout = sdhci_target_timeout(host, cmd, data);
 955
 956        /*
 957         * Figure out needed cycles.
 958         * We do this in steps in order to fit inside a 32 bit int.
 959         * The first step is the minimum timeout, which will have a
 960         * minimum resolution of 6 bits:
 961         * (1) 2^13*1000 > 2^22,
 962         * (2) host->timeout_clk < 2^16
 963         *     =>
 964         *     (1) / (2) > 2^6
 965         */
 966        count = 0;
 967        current_timeout = (1 << 13) * 1000 / host->timeout_clk;
 968        while (current_timeout < target_timeout) {
 969                count++;
 970                current_timeout <<= 1;
 971                if (count >= 0xF)
 972                        break;
 973        }
 974
 975        if (count >= 0xF) {
 976                if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))
 977                        DBG("Too large timeout 0x%x requested for CMD%d!\n",
 978                            count, cmd->opcode);
 979                count = 0xE;
 980        } else {
 981                *too_big = false;
 982        }
 983
 984        return count;
 985}
 986
 987static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 988{
 989        u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
 990        u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
 991
 992        if (host->flags & SDHCI_REQ_USE_DMA)
 993                host->ier = (host->ier & ~pio_irqs) | dma_irqs;
 994        else
 995                host->ier = (host->ier & ~dma_irqs) | pio_irqs;
 996
 997        if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12))
 998                host->ier |= SDHCI_INT_AUTO_CMD_ERR;
 999        else
1000                host->ier &= ~SDHCI_INT_AUTO_CMD_ERR;
1001
1002        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1003        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1004}
1005
1006void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable)
1007{
1008        if (enable)
1009                host->ier |= SDHCI_INT_DATA_TIMEOUT;
1010        else
1011                host->ier &= ~SDHCI_INT_DATA_TIMEOUT;
1012        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1013        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1014}
1015EXPORT_SYMBOL_GPL(sdhci_set_data_timeout_irq);
1016
1017void __sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
1018{
1019        bool too_big = false;
1020        u8 count = sdhci_calc_timeout(host, cmd, &too_big);
1021
1022        if (too_big &&
1023            host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {
1024                sdhci_calc_sw_timeout(host, cmd);
1025                sdhci_set_data_timeout_irq(host, false);
1026        } else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {
1027                sdhci_set_data_timeout_irq(host, true);
1028        }
1029
1030        sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
1031}
1032EXPORT_SYMBOL_GPL(__sdhci_set_timeout);
1033
1034static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
1035{
1036        if (host->ops->set_timeout)
1037                host->ops->set_timeout(host, cmd);
1038        else
1039                __sdhci_set_timeout(host, cmd);
1040}
1041
1042static void sdhci_initialize_data(struct sdhci_host *host,
1043                                  struct mmc_data *data)
1044{
1045        WARN_ON(host->data);
1046
1047        /* Sanity checks */
1048        BUG_ON(data->blksz * data->blocks > 524288);
1049        BUG_ON(data->blksz > host->mmc->max_blk_size);
1050        BUG_ON(data->blocks > 65535);
1051
1052        host->data = data;
1053        host->data_early = 0;
1054        host->data->bytes_xfered = 0;
1055}
1056
1057static inline void sdhci_set_block_info(struct sdhci_host *host,
1058                                        struct mmc_data *data)
1059{
1060        /* Set the DMA boundary value and block size */
1061        sdhci_writew(host,
1062                     SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
1063                     SDHCI_BLOCK_SIZE);
1064        /*
1065         * For Version 4.10 onwards, if v4 mode is enabled, 32-bit Block Count
1066         * can be supported, in that case 16-bit block count register must be 0.
1067         */
1068        if (host->version >= SDHCI_SPEC_410 && host->v4_mode &&
1069            (host->quirks2 & SDHCI_QUIRK2_USE_32BIT_BLK_CNT)) {
1070                if (sdhci_readw(host, SDHCI_BLOCK_COUNT))
1071                        sdhci_writew(host, 0, SDHCI_BLOCK_COUNT);
1072                sdhci_writew(host, data->blocks, SDHCI_32BIT_BLK_CNT);
1073        } else {
1074                sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
1075        }
1076}
1077
1078static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
1079{
1080        struct mmc_data *data = cmd->data;
1081
1082        sdhci_initialize_data(host, data);
1083
1084        if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
1085                struct scatterlist *sg;
1086                unsigned int length_mask, offset_mask;
1087                int i;
1088
1089                host->flags |= SDHCI_REQ_USE_DMA;
1090
1091                /*
1092                 * FIXME: This doesn't account for merging when mapping the
1093                 * scatterlist.
1094                 *
1095                 * The assumption here being that alignment and lengths are
1096                 * the same after DMA mapping to device address space.
1097                 */
1098                length_mask = 0;
1099                offset_mask = 0;
1100                if (host->flags & SDHCI_USE_ADMA) {
1101                        if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
1102                                length_mask = 3;
1103                                /*
1104                                 * As we use up to 3 byte chunks to work
1105                                 * around alignment problems, we need to
1106                                 * check the offset as well.
1107                                 */
1108                                offset_mask = 3;
1109                        }
1110                } else {
1111                        if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
1112                                length_mask = 3;
1113                        if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
1114                                offset_mask = 3;
1115                }
1116
1117                if (unlikely(length_mask | offset_mask)) {
1118                        for_each_sg(data->sg, sg, data->sg_len, i) {
1119                                if (sg->length & length_mask) {
1120                                        DBG("Reverting to PIO because of transfer size (%d)\n",
1121                                            sg->length);
1122                                        host->flags &= ~SDHCI_REQ_USE_DMA;
1123                                        break;
1124                                }
1125                                if (sg->offset & offset_mask) {
1126                                        DBG("Reverting to PIO because of bad alignment\n");
1127                                        host->flags &= ~SDHCI_REQ_USE_DMA;
1128                                        break;
1129                                }
1130                        }
1131                }
1132        }
1133
1134        if (host->flags & SDHCI_REQ_USE_DMA) {
1135                int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
1136
1137                if (sg_cnt <= 0) {
1138                        /*
1139                         * This only happens when someone fed
1140                         * us an invalid request.
1141                         */
1142                        WARN_ON(1);
1143                        host->flags &= ~SDHCI_REQ_USE_DMA;
1144                } else if (host->flags & SDHCI_USE_ADMA) {
1145                        sdhci_adma_table_pre(host, data, sg_cnt);
1146                        sdhci_set_adma_addr(host, host->adma_addr);
1147                } else {
1148                        WARN_ON(sg_cnt != 1);
1149                        sdhci_set_sdma_addr(host, sdhci_sdma_address(host));
1150                }
1151        }
1152
1153        sdhci_config_dma(host);
1154
1155        if (!(host->flags & SDHCI_REQ_USE_DMA)) {
1156                int flags;
1157
1158                flags = SG_MITER_ATOMIC;
1159                if (host->data->flags & MMC_DATA_READ)
1160                        flags |= SG_MITER_TO_SG;
1161                else
1162                        flags |= SG_MITER_FROM_SG;
1163                sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
1164                host->blocks = data->blocks;
1165        }
1166
1167        sdhci_set_transfer_irqs(host);
1168
1169        sdhci_set_block_info(host, data);
1170}
1171
1172#if IS_ENABLED(CONFIG_MMC_SDHCI_EXTERNAL_DMA)
1173
1174static int sdhci_external_dma_init(struct sdhci_host *host)
1175{
1176        int ret = 0;
1177        struct mmc_host *mmc = host->mmc;
1178
1179        host->tx_chan = dma_request_chan(mmc_dev(mmc), "tx");
1180        if (IS_ERR(host->tx_chan)) {
1181                ret = PTR_ERR(host->tx_chan);
1182                if (ret != -EPROBE_DEFER)
1183                        pr_warn("Failed to request TX DMA channel.\n");
1184                host->tx_chan = NULL;
1185                return ret;
1186        }
1187
1188        host->rx_chan = dma_request_chan(mmc_dev(mmc), "rx");
1189        if (IS_ERR(host->rx_chan)) {
1190                if (host->tx_chan) {
1191                        dma_release_channel(host->tx_chan);
1192                        host->tx_chan = NULL;
1193                }
1194
1195                ret = PTR_ERR(host->rx_chan);
1196                if (ret != -EPROBE_DEFER)
1197                        pr_warn("Failed to request RX DMA channel.\n");
1198                host->rx_chan = NULL;
1199        }
1200
1201        return ret;
1202}
1203
1204static struct dma_chan *sdhci_external_dma_channel(struct sdhci_host *host,
1205                                                   struct mmc_data *data)
1206{
1207        return data->flags & MMC_DATA_WRITE ? host->tx_chan : host->rx_chan;
1208}
1209
1210static int sdhci_external_dma_setup(struct sdhci_host *host,
1211                                    struct mmc_command *cmd)
1212{
1213        int ret, i;
1214        enum dma_transfer_direction dir;
1215        struct dma_async_tx_descriptor *desc;
1216        struct mmc_data *data = cmd->data;
1217        struct dma_chan *chan;
1218        struct dma_slave_config cfg;
1219        dma_cookie_t cookie;
1220        int sg_cnt;
1221
1222        if (!host->mapbase)
1223                return -EINVAL;
1224
1225        cfg.src_addr = host->mapbase + SDHCI_BUFFER;
1226        cfg.dst_addr = host->mapbase + SDHCI_BUFFER;
1227        cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1228        cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1229        cfg.src_maxburst = data->blksz / 4;
1230        cfg.dst_maxburst = data->blksz / 4;
1231
1232        /* Sanity check: all the SG entries must be aligned by block size. */
1233        for (i = 0; i < data->sg_len; i++) {
1234                if ((data->sg + i)->length % data->blksz)
1235                        return -EINVAL;
1236        }
1237
1238        chan = sdhci_external_dma_channel(host, data);
1239
1240        ret = dmaengine_slave_config(chan, &cfg);
1241        if (ret)
1242                return ret;
1243
1244        sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
1245        if (sg_cnt <= 0)
1246                return -EINVAL;
1247
1248        dir = data->flags & MMC_DATA_WRITE ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
1249        desc = dmaengine_prep_slave_sg(chan, data->sg, data->sg_len, dir,
1250                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1251        if (!desc)
1252                return -EINVAL;
1253
1254        desc->callback = NULL;
1255        desc->callback_param = NULL;
1256
1257        cookie = dmaengine_submit(desc);
1258        if (dma_submit_error(cookie))
1259                ret = cookie;
1260
1261        return ret;
1262}
1263
1264static void sdhci_external_dma_release(struct sdhci_host *host)
1265{
1266        if (host->tx_chan) {
1267                dma_release_channel(host->tx_chan);
1268                host->tx_chan = NULL;
1269        }
1270
1271        if (host->rx_chan) {
1272                dma_release_channel(host->rx_chan);
1273                host->rx_chan = NULL;
1274        }
1275
1276        sdhci_switch_external_dma(host, false);
1277}
1278
1279static void __sdhci_external_dma_prepare_data(struct sdhci_host *host,
1280                                              struct mmc_command *cmd)
1281{
1282        struct mmc_data *data = cmd->data;
1283
1284        sdhci_initialize_data(host, data);
1285
1286        host->flags |= SDHCI_REQ_USE_DMA;
1287        sdhci_set_transfer_irqs(host);
1288
1289        sdhci_set_block_info(host, data);
1290}
1291
1292static void sdhci_external_dma_prepare_data(struct sdhci_host *host,
1293                                            struct mmc_command *cmd)
1294{
1295        if (!sdhci_external_dma_setup(host, cmd)) {
1296                __sdhci_external_dma_prepare_data(host, cmd);
1297        } else {
1298                sdhci_external_dma_release(host);
1299                pr_err("%s: Cannot use external DMA, switch to the DMA/PIO which standard SDHCI provides.\n",
1300                       mmc_hostname(host->mmc));
1301                sdhci_prepare_data(host, cmd);
1302        }
1303}
1304
1305static void sdhci_external_dma_pre_transfer(struct sdhci_host *host,
1306                                            struct mmc_command *cmd)
1307{
1308        struct dma_chan *chan;
1309
1310        if (!cmd->data)
1311                return;
1312
1313        chan = sdhci_external_dma_channel(host, cmd->data);
1314        if (chan)
1315                dma_async_issue_pending(chan);
1316}
1317
1318#else
1319
1320static inline int sdhci_external_dma_init(struct sdhci_host *host)
1321{
1322        return -EOPNOTSUPP;
1323}
1324
1325static inline void sdhci_external_dma_release(struct sdhci_host *host)
1326{
1327}
1328
1329static inline void sdhci_external_dma_prepare_data(struct sdhci_host *host,
1330                                                   struct mmc_command *cmd)
1331{
1332        /* This should never happen */
1333        WARN_ON_ONCE(1);
1334}
1335
1336static inline void sdhci_external_dma_pre_transfer(struct sdhci_host *host,
1337                                                   struct mmc_command *cmd)
1338{
1339}
1340
1341static inline struct dma_chan *sdhci_external_dma_channel(struct sdhci_host *host,
1342                                                          struct mmc_data *data)
1343{
1344        return NULL;
1345}
1346
1347#endif
1348
1349void sdhci_switch_external_dma(struct sdhci_host *host, bool en)
1350{
1351        host->use_external_dma = en;
1352}
1353EXPORT_SYMBOL_GPL(sdhci_switch_external_dma);
1354
1355static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
1356                                    struct mmc_request *mrq)
1357{
1358        return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
1359               !mrq->cap_cmd_during_tfr;
1360}
1361
1362static inline bool sdhci_auto_cmd23(struct sdhci_host *host,
1363                                    struct mmc_request *mrq)
1364{
1365        return mrq->sbc && (host->flags & SDHCI_AUTO_CMD23);
1366}
1367
1368static inline bool sdhci_manual_cmd23(struct sdhci_host *host,
1369                                      struct mmc_request *mrq)
1370{
1371        return mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23);
1372}
1373
1374static inline void sdhci_auto_cmd_select(struct sdhci_host *host,
1375                                         struct mmc_command *cmd,
1376                                         u16 *mode)
1377{
1378        bool use_cmd12 = sdhci_auto_cmd12(host, cmd->mrq) &&
1379                         (cmd->opcode != SD_IO_RW_EXTENDED);
1380        bool use_cmd23 = sdhci_auto_cmd23(host, cmd->mrq);
1381        u16 ctrl2;
1382
1383        /*
1384         * In case of Version 4.10 or later, use of 'Auto CMD Auto
1385         * Select' is recommended rather than use of 'Auto CMD12
1386         * Enable' or 'Auto CMD23 Enable'. We require Version 4 Mode
1387         * here because some controllers (e.g sdhci-of-dwmshc) expect it.
1388         */
1389        if (host->version >= SDHCI_SPEC_410 && host->v4_mode &&
1390            (use_cmd12 || use_cmd23)) {
1391                *mode |= SDHCI_TRNS_AUTO_SEL;
1392
1393                ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1394                if (use_cmd23)
1395                        ctrl2 |= SDHCI_CMD23_ENABLE;
1396                else
1397                        ctrl2 &= ~SDHCI_CMD23_ENABLE;
1398                sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
1399
1400                return;
1401        }
1402
1403        /*
1404         * If we are sending CMD23, CMD12 never gets sent
1405         * on successful completion (so no Auto-CMD12).
1406         */
1407        if (use_cmd12)
1408                *mode |= SDHCI_TRNS_AUTO_CMD12;
1409        else if (use_cmd23)
1410                *mode |= SDHCI_TRNS_AUTO_CMD23;
1411}
1412
1413static void sdhci_set_transfer_mode(struct sdhci_host *host,
1414        struct mmc_command *cmd)
1415{
1416        u16 mode = 0;
1417        struct mmc_data *data = cmd->data;
1418
1419        if (data == NULL) {
1420                if (host->quirks2 &
1421                        SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
1422                        /* must not clear SDHCI_TRANSFER_MODE when tuning */
1423                        if (cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200)
1424                                sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
1425                } else {
1426                /* clear Auto CMD settings for no data CMDs */
1427                        mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
1428                        sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
1429                                SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
1430                }
1431                return;
1432        }
1433
1434        WARN_ON(!host->data);
1435
1436        if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
1437                mode = SDHCI_TRNS_BLK_CNT_EN;
1438
1439        if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
1440                mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
1441                sdhci_auto_cmd_select(host, cmd, &mode);
1442                if (sdhci_auto_cmd23(host, cmd->mrq))
1443                        sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
1444        }
1445
1446        if (data->flags & MMC_DATA_READ)
1447                mode |= SDHCI_TRNS_READ;
1448        if (host->flags & SDHCI_REQ_USE_DMA)
1449                mode |= SDHCI_TRNS_DMA;
1450
1451        sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1452}
1453
1454static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1455{
1456        return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1457                ((mrq->cmd && mrq->cmd->error) ||
1458                 (mrq->sbc && mrq->sbc->error) ||
1459                 (mrq->data && mrq->data->stop && mrq->data->stop->error) ||
1460                 (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1461}
1462
1463static void sdhci_set_mrq_done(struct sdhci_host *host, struct mmc_request *mrq)
1464{
1465        int i;
1466
1467        for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1468                if (host->mrqs_done[i] == mrq) {
1469                        WARN_ON(1);
1470                        return;
1471                }
1472        }
1473
1474        for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1475                if (!host->mrqs_done[i]) {
1476                        host->mrqs_done[i] = mrq;
1477                        break;
1478                }
1479        }
1480
1481        WARN_ON(i >= SDHCI_MAX_MRQS);
1482}
1483
1484static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1485{
1486        if (host->cmd && host->cmd->mrq == mrq)
1487                host->cmd = NULL;
1488
1489        if (host->data_cmd && host->data_cmd->mrq == mrq)
1490                host->data_cmd = NULL;
1491
1492        if (host->deferred_cmd && host->deferred_cmd->mrq == mrq)
1493                host->deferred_cmd = NULL;
1494
1495        if (host->data && host->data->mrq == mrq)
1496                host->data = NULL;
1497
1498        if (sdhci_needs_reset(host, mrq))
1499                host->pending_reset = true;
1500
1501        sdhci_set_mrq_done(host, mrq);
1502
1503        sdhci_del_timer(host, mrq);
1504
1505        if (!sdhci_has_requests(host))
1506                sdhci_led_deactivate(host);
1507}
1508
1509static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1510{
1511        __sdhci_finish_mrq(host, mrq);
1512
1513        queue_work(host->complete_wq, &host->complete_work);
1514}
1515
1516static void __sdhci_finish_data(struct sdhci_host *host, bool sw_data_timeout)
1517{
1518        struct mmc_command *data_cmd = host->data_cmd;
1519        struct mmc_data *data = host->data;
1520
1521        host->data = NULL;
1522        host->data_cmd = NULL;
1523
1524        /*
1525         * The controller needs a reset of internal state machines upon error
1526         * conditions.
1527         */
1528        if (data->error) {
1529                if (!host->cmd || host->cmd == data_cmd)
1530                        sdhci_do_reset(host, SDHCI_RESET_CMD);
1531                sdhci_do_reset(host, SDHCI_RESET_DATA);
1532        }
1533
1534        if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1535            (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1536                sdhci_adma_table_post(host, data);
1537
1538        /*
1539         * The specification states that the block count register must
1540         * be updated, but it does not specify at what point in the
1541         * data flow. That makes the register entirely useless to read
1542         * back so we have to assume that nothing made it to the card
1543         * in the event of an error.
1544         */
1545        if (data->error)
1546                data->bytes_xfered = 0;
1547        else
1548                data->bytes_xfered = data->blksz * data->blocks;
1549
1550        /*
1551         * Need to send CMD12 if -
1552         * a) open-ended multiblock transfer not using auto CMD12 (no CMD23)
1553         * b) error in multiblock transfer
1554         */
1555        if (data->stop &&
1556            ((!data->mrq->sbc && !sdhci_auto_cmd12(host, data->mrq)) ||
1557             data->error)) {
1558                /*
1559                 * 'cap_cmd_during_tfr' request must not use the command line
1560                 * after mmc_command_done() has been called. It is upper layer's
1561                 * responsibility to send the stop command if required.
1562                 */
1563                if (data->mrq->cap_cmd_during_tfr) {
1564                        __sdhci_finish_mrq(host, data->mrq);
1565                } else {
1566                        /* Avoid triggering warning in sdhci_send_command() */
1567                        host->cmd = NULL;
1568                        if (!sdhci_send_command(host, data->stop)) {
1569                                if (sw_data_timeout) {
1570                                        /*
1571                                         * This is anyway a sw data timeout, so
1572                                         * give up now.
1573                                         */
1574                                        data->stop->error = -EIO;
1575                                        __sdhci_finish_mrq(host, data->mrq);
1576                                } else {
1577                                        WARN_ON(host->deferred_cmd);
1578                                        host->deferred_cmd = data->stop;
1579                                }
1580                        }
1581                }
1582        } else {
1583                __sdhci_finish_mrq(host, data->mrq);
1584        }
1585}
1586
1587static void sdhci_finish_data(struct sdhci_host *host)
1588{
1589        __sdhci_finish_data(host, false);
1590}
1591
1592static bool sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1593{
1594        int flags;
1595        u32 mask;
1596        unsigned long timeout;
1597
1598        WARN_ON(host->cmd);
1599
1600        /* Initially, a command has no error */
1601        cmd->error = 0;
1602
1603        if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1604            cmd->opcode == MMC_STOP_TRANSMISSION)
1605                cmd->flags |= MMC_RSP_BUSY;
1606
1607        mask = SDHCI_CMD_INHIBIT;
1608        if (sdhci_data_line_cmd(cmd))
1609                mask |= SDHCI_DATA_INHIBIT;
1610
1611        /* We shouldn't wait for data inihibit for stop commands, even
1612           though they might use busy signaling */
1613        if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1614                mask &= ~SDHCI_DATA_INHIBIT;
1615
1616        if (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask)
1617                return false;
1618
1619        host->cmd = cmd;
1620        host->data_timeout = 0;
1621        if (sdhci_data_line_cmd(cmd)) {
1622                WARN_ON(host->data_cmd);
1623                host->data_cmd = cmd;
1624                sdhci_set_timeout(host, cmd);
1625        }
1626
1627        if (cmd->data) {
1628                if (host->use_external_dma)
1629                        sdhci_external_dma_prepare_data(host, cmd);
1630                else
1631                        sdhci_prepare_data(host, cmd);
1632        }
1633
1634        sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1635
1636        sdhci_set_transfer_mode(host, cmd);
1637
1638        if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1639                WARN_ONCE(1, "Unsupported response type!\n");
1640                /*
1641                 * This does not happen in practice because 136-bit response
1642                 * commands never have busy waiting, so rather than complicate
1643                 * the error path, just remove busy waiting and continue.
1644                 */
1645                cmd->flags &= ~MMC_RSP_BUSY;
1646        }
1647
1648        if (!(cmd->flags & MMC_RSP_PRESENT))
1649                flags = SDHCI_CMD_RESP_NONE;
1650        else if (cmd->flags & MMC_RSP_136)
1651                flags = SDHCI_CMD_RESP_LONG;
1652        else if (cmd->flags & MMC_RSP_BUSY)
1653                flags = SDHCI_CMD_RESP_SHORT_BUSY;
1654        else
1655                flags = SDHCI_CMD_RESP_SHORT;
1656
1657        if (cmd->flags & MMC_RSP_CRC)
1658                flags |= SDHCI_CMD_CRC;
1659        if (cmd->flags & MMC_RSP_OPCODE)
1660                flags |= SDHCI_CMD_INDEX;
1661
1662        /* CMD19 is special in that the Data Present Select should be set */
1663        if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1664            cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1665                flags |= SDHCI_CMD_DATA;
1666
1667        timeout = jiffies;
1668        if (host->data_timeout)
1669                timeout += nsecs_to_jiffies(host->data_timeout);
1670        else if (!cmd->data && cmd->busy_timeout > 9000)
1671                timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1672        else
1673                timeout += 10 * HZ;
1674        sdhci_mod_timer(host, cmd->mrq, timeout);
1675
1676        if (host->use_external_dma)
1677                sdhci_external_dma_pre_transfer(host, cmd);
1678
1679        sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1680
1681        return true;
1682}
1683
1684static bool sdhci_present_error(struct sdhci_host *host,
1685                                struct mmc_command *cmd, bool present)
1686{
1687        if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1688                cmd->error = -ENOMEDIUM;
1689                return true;
1690        }
1691
1692        return false;
1693}
1694
1695static bool sdhci_send_command_retry(struct sdhci_host *host,
1696                                     struct mmc_command *cmd,
1697                                     unsigned long flags)
1698        __releases(host->lock)
1699        __acquires(host->lock)
1700{
1701        struct mmc_command *deferred_cmd = host->deferred_cmd;
1702        int timeout = 10; /* Approx. 10 ms */
1703        bool present;
1704
1705        while (!sdhci_send_command(host, cmd)) {
1706                if (!timeout--) {
1707                        pr_err("%s: Controller never released inhibit bit(s).\n",
1708                               mmc_hostname(host->mmc));
1709                        sdhci_dumpregs(host);
1710                        cmd->error = -EIO;
1711                        return false;
1712                }
1713
1714                spin_unlock_irqrestore(&host->lock, flags);
1715
1716                usleep_range(1000, 1250);
1717
1718                present = host->mmc->ops->get_cd(host->mmc);
1719
1720                spin_lock_irqsave(&host->lock, flags);
1721
1722                /* A deferred command might disappear, handle that */
1723                if (cmd == deferred_cmd && cmd != host->deferred_cmd)
1724                        return true;
1725
1726                if (sdhci_present_error(host, cmd, present))
1727                        return false;
1728        }
1729
1730        if (cmd == host->deferred_cmd)
1731                host->deferred_cmd = NULL;
1732
1733        return true;
1734}
1735
1736static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1737{
1738        int i, reg;
1739
1740        for (i = 0; i < 4; i++) {
1741                reg = SDHCI_RESPONSE + (3 - i) * 4;
1742                cmd->resp[i] = sdhci_readl(host, reg);
1743        }
1744
1745        if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1746                return;
1747
1748        /* CRC is stripped so we need to do some shifting */
1749        for (i = 0; i < 4; i++) {
1750                cmd->resp[i] <<= 8;
1751                if (i != 3)
1752                        cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1753        }
1754}
1755
1756static void sdhci_finish_command(struct sdhci_host *host)
1757{
1758        struct mmc_command *cmd = host->cmd;
1759
1760        host->cmd = NULL;
1761
1762        if (cmd->flags & MMC_RSP_PRESENT) {
1763                if (cmd->flags & MMC_RSP_136) {
1764                        sdhci_read_rsp_136(host, cmd);
1765                } else {
1766                        cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1767                }
1768        }
1769
1770        if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1771                mmc_command_done(host->mmc, cmd->mrq);
1772
1773        /*
1774         * The host can send and interrupt when the busy state has
1775         * ended, allowing us to wait without wasting CPU cycles.
1776         * The busy signal uses DAT0 so this is similar to waiting
1777         * for data to complete.
1778         *
1779         * Note: The 1.0 specification is a bit ambiguous about this
1780         *       feature so there might be some problems with older
1781         *       controllers.
1782         */
1783        if (cmd->flags & MMC_RSP_BUSY) {
1784                if (cmd->data) {
1785                        DBG("Cannot wait for busy signal when also doing a data transfer");
1786                } else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1787                           cmd == host->data_cmd) {
1788                        /* Command complete before busy is ended */
1789                        return;
1790                }
1791        }
1792
1793        /* Finished CMD23, now send actual command. */
1794        if (cmd == cmd->mrq->sbc) {
1795                if (!sdhci_send_command(host, cmd->mrq->cmd)) {
1796                        WARN_ON(host->deferred_cmd);
1797                        host->deferred_cmd = cmd->mrq->cmd;
1798                }
1799        } else {
1800
1801                /* Processed actual command. */
1802                if (host->data && host->data_early)
1803                        sdhci_finish_data(host);
1804
1805                if (!cmd->data)
1806                        __sdhci_finish_mrq(host, cmd->mrq);
1807        }
1808}
1809
1810static u16 sdhci_get_preset_value(struct sdhci_host *host)
1811{
1812        u16 preset = 0;
1813
1814        switch (host->timing) {
1815        case MMC_TIMING_MMC_HS:
1816        case MMC_TIMING_SD_HS:
1817                preset = sdhci_readw(host, SDHCI_PRESET_FOR_HIGH_SPEED);
1818                break;
1819        case MMC_TIMING_UHS_SDR12:
1820                preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1821                break;
1822        case MMC_TIMING_UHS_SDR25:
1823                preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1824                break;
1825        case MMC_TIMING_UHS_SDR50:
1826                preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1827                break;
1828        case MMC_TIMING_UHS_SDR104:
1829        case MMC_TIMING_MMC_HS200:
1830                preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1831                break;
1832        case MMC_TIMING_UHS_DDR50:
1833        case MMC_TIMING_MMC_DDR52:
1834                preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1835                break;
1836        case MMC_TIMING_MMC_HS400:
1837                preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1838                break;
1839        default:
1840                pr_warn("%s: Invalid UHS-I mode selected\n",
1841                        mmc_hostname(host->mmc));
1842                preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1843                break;
1844        }
1845        return preset;
1846}
1847
1848u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1849                   unsigned int *actual_clock)
1850{
1851        int div = 0; /* Initialized for compiler warning */
1852        int real_div = div, clk_mul = 1;
1853        u16 clk = 0;
1854        bool switch_base_clk = false;
1855
1856        if (host->version >= SDHCI_SPEC_300) {
1857                if (host->preset_enabled) {
1858                        u16 pre_val;
1859
1860                        clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1861                        pre_val = sdhci_get_preset_value(host);
1862                        div = FIELD_GET(SDHCI_PRESET_SDCLK_FREQ_MASK, pre_val);
1863                        if (host->clk_mul &&
1864                                (pre_val & SDHCI_PRESET_CLKGEN_SEL)) {
1865                                clk = SDHCI_PROG_CLOCK_MODE;
1866                                real_div = div + 1;
1867                                clk_mul = host->clk_mul;
1868                        } else {
1869                                real_div = max_t(int, 1, div << 1);
1870                        }
1871                        goto clock_set;
1872                }
1873
1874                /*
1875                 * Check if the Host Controller supports Programmable Clock
1876                 * Mode.
1877                 */
1878                if (host->clk_mul) {
1879                        for (div = 1; div <= 1024; div++) {
1880                                if ((host->max_clk * host->clk_mul / div)
1881                                        <= clock)
1882                                        break;
1883                        }
1884                        if ((host->max_clk * host->clk_mul / div) <= clock) {
1885                                /*
1886                                 * Set Programmable Clock Mode in the Clock
1887                                 * Control register.
1888                                 */
1889                                clk = SDHCI_PROG_CLOCK_MODE;
1890                                real_div = div;
1891                                clk_mul = host->clk_mul;
1892                                div--;
1893                        } else {
1894                                /*
1895                                 * Divisor can be too small to reach clock
1896                                 * speed requirement. Then use the base clock.
1897                                 */
1898                                switch_base_clk = true;
1899                        }
1900                }
1901
1902                if (!host->clk_mul || switch_base_clk) {
1903                        /* Version 3.00 divisors must be a multiple of 2. */
1904                        if (host->max_clk <= clock)
1905                                div = 1;
1906                        else {
1907                                for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1908                                     div += 2) {
1909                                        if ((host->max_clk / div) <= clock)
1910                                                break;
1911                                }
1912                        }
1913                        real_div = div;
1914                        div >>= 1;
1915                        if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1916                                && !div && host->max_clk <= 25000000)
1917                                div = 1;
1918                }
1919        } else {
1920                /* Version 2.00 divisors must be a power of 2. */
1921                for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1922                        if ((host->max_clk / div) <= clock)
1923                                break;
1924                }
1925                real_div = div;
1926                div >>= 1;
1927        }
1928
1929clock_set:
1930        if (real_div)
1931                *actual_clock = (host->max_clk * clk_mul) / real_div;
1932        clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1933        clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1934                << SDHCI_DIVIDER_HI_SHIFT;
1935
1936        return clk;
1937}
1938EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1939
1940void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1941{
1942        ktime_t timeout;
1943
1944        clk |= SDHCI_CLOCK_INT_EN;
1945        sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1946
1947        /* Wait max 150 ms */
1948        timeout = ktime_add_ms(ktime_get(), 150);
1949        while (1) {
1950                bool timedout = ktime_after(ktime_get(), timeout);
1951
1952                clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1953                if (clk & SDHCI_CLOCK_INT_STABLE)
1954                        break;
1955                if (timedout) {
1956                        pr_err("%s: Internal clock never stabilised.\n",
1957                               mmc_hostname(host->mmc));
1958                        sdhci_dumpregs(host);
1959                        return;
1960                }
1961                udelay(10);
1962        }
1963
1964        if (host->version >= SDHCI_SPEC_410 && host->v4_mode) {
1965                clk |= SDHCI_CLOCK_PLL_EN;
1966                clk &= ~SDHCI_CLOCK_INT_STABLE;
1967                sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1968
1969                /* Wait max 150 ms */
1970                timeout = ktime_add_ms(ktime_get(), 150);
1971                while (1) {
1972                        bool timedout = ktime_after(ktime_get(), timeout);
1973
1974                        clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1975                        if (clk & SDHCI_CLOCK_INT_STABLE)
1976                                break;
1977                        if (timedout) {
1978                                pr_err("%s: PLL clock never stabilised.\n",
1979                                       mmc_hostname(host->mmc));
1980                                sdhci_dumpregs(host);
1981                                return;
1982                        }
1983                        udelay(10);
1984                }
1985        }
1986
1987        clk |= SDHCI_CLOCK_CARD_EN;
1988        sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1989}
1990EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1991
1992void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1993{
1994        u16 clk;
1995
1996        host->mmc->actual_clock = 0;
1997
1998        sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1999
2000        if (clock == 0)
2001                return;
2002
2003        clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
2004        sdhci_enable_clk(host, clk);
2005}
2006EXPORT_SYMBOL_GPL(sdhci_set_clock);
2007
2008static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
2009                                unsigned short vdd)
2010{
2011        struct mmc_host *mmc = host->mmc;
2012
2013        mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
2014
2015        if (mode != MMC_POWER_OFF)
2016                sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
2017        else
2018                sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
2019}
2020
2021void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
2022                           unsigned short vdd)
2023{
2024        u8 pwr = 0;
2025
2026        if (mode != MMC_POWER_OFF) {
2027                switch (1 << vdd) {
2028                case MMC_VDD_165_195:
2029                /*
2030                 * Without a regulator, SDHCI does not support 2.0v
2031                 * so we only get here if the driver deliberately
2032                 * added the 2.0v range to ocr_avail. Map it to 1.8v
2033                 * for the purpose of turning on the power.
2034                 */
2035                case MMC_VDD_20_21:
2036                        pwr = SDHCI_POWER_180;
2037                        break;
2038                case MMC_VDD_29_30:
2039                case MMC_VDD_30_31:
2040                        pwr = SDHCI_POWER_300;
2041                        break;
2042                case MMC_VDD_32_33:
2043                case MMC_VDD_33_34:
2044                        pwr = SDHCI_POWER_330;
2045                        break;
2046                default:
2047                        WARN(1, "%s: Invalid vdd %#x\n",
2048                             mmc_hostname(host->mmc), vdd);
2049                        break;
2050                }
2051        }
2052
2053        if (host->pwr == pwr)
2054                return;
2055
2056        host->pwr = pwr;
2057
2058        if (pwr == 0) {
2059                sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
2060                if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
2061                        sdhci_runtime_pm_bus_off(host);
2062        } else {
2063                /*
2064                 * Spec says that we should clear the power reg before setting
2065                 * a new value. Some controllers don't seem to like this though.
2066                 */
2067                if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
2068                        sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
2069
2070                /*
2071                 * At least the Marvell CaFe chip gets confused if we set the
2072                 * voltage and set turn on power at the same time, so set the
2073                 * voltage first.
2074                 */
2075                if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
2076                        sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
2077
2078                pwr |= SDHCI_POWER_ON;
2079
2080                sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
2081
2082                if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
2083                        sdhci_runtime_pm_bus_on(host);
2084
2085                /*
2086                 * Some controllers need an extra 10ms delay of 10ms before
2087                 * they can apply clock after applying power
2088                 */
2089                if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
2090                        mdelay(10);
2091        }
2092}
2093EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
2094
2095void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
2096                     unsigned short vdd)
2097{
2098        if (IS_ERR(host->mmc->supply.vmmc))
2099                sdhci_set_power_noreg(host, mode, vdd);
2100        else
2101                sdhci_set_power_reg(host, mode, vdd);
2102}
2103EXPORT_SYMBOL_GPL(sdhci_set_power);
2104
2105/*
2106 * Some controllers need to configure a valid bus voltage on their power
2107 * register regardless of whether an external regulator is taking care of power
2108 * supply. This helper function takes care of it if set as the controller's
2109 * sdhci_ops.set_power callback.
2110 */
2111void sdhci_set_power_and_bus_voltage(struct sdhci_host *host,
2112                                     unsigned char mode,
2113                                     unsigned short vdd)
2114{
2115        if (!IS_ERR(host->mmc->supply.vmmc)) {
2116                struct mmc_host *mmc = host->mmc;
2117
2118                mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
2119        }
2120        sdhci_set_power_noreg(host, mode, vdd);
2121}
2122EXPORT_SYMBOL_GPL(sdhci_set_power_and_bus_voltage);
2123
2124/*****************************************************************************\
2125 *                                                                           *
2126 * MMC callbacks                                                             *
2127 *                                                                           *
2128\*****************************************************************************/
2129
2130void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
2131{
2132        struct sdhci_host *host = mmc_priv(mmc);
2133        struct mmc_command *cmd;
2134        unsigned long flags;
2135        bool present;
2136
2137        /* Firstly check card presence */
2138        present = mmc->ops->get_cd(mmc);
2139
2140        spin_lock_irqsave(&host->lock, flags);
2141
2142        sdhci_led_activate(host);
2143
2144        if (sdhci_present_error(host, mrq->cmd, present))
2145                goto out_finish;
2146
2147        cmd = sdhci_manual_cmd23(host, mrq) ? mrq->sbc : mrq->cmd;
2148
2149        if (!sdhci_send_command_retry(host, cmd, flags))
2150                goto out_finish;
2151
2152        spin_unlock_irqrestore(&host->lock, flags);
2153
2154        return;
2155
2156out_finish:
2157        sdhci_finish_mrq(host, mrq);
2158        spin_unlock_irqrestore(&host->lock, flags);
2159}
2160EXPORT_SYMBOL_GPL(sdhci_request);
2161
2162int sdhci_request_atomic(struct mmc_host *mmc, struct mmc_request *mrq)
2163{
2164        struct sdhci_host *host = mmc_priv(mmc);
2165        struct mmc_command *cmd;
2166        unsigned long flags;
2167        int ret = 0;
2168
2169        spin_lock_irqsave(&host->lock, flags);
2170
2171        if (sdhci_present_error(host, mrq->cmd, true)) {
2172                sdhci_finish_mrq(host, mrq);
2173                goto out_finish;
2174        }
2175
2176        cmd = sdhci_manual_cmd23(host, mrq) ? mrq->sbc : mrq->cmd;
2177
2178        /*
2179         * The HSQ may send a command in interrupt context without polling
2180         * the busy signaling, which means we should return BUSY if controller
2181         * has not released inhibit bits to allow HSQ trying to send request
2182         * again in non-atomic context. So we should not finish this request
2183         * here.
2184         */
2185        if (!sdhci_send_command(host, cmd))
2186                ret = -EBUSY;
2187        else
2188                sdhci_led_activate(host);
2189
2190out_finish:
2191        spin_unlock_irqrestore(&host->lock, flags);
2192        return ret;
2193}
2194EXPORT_SYMBOL_GPL(sdhci_request_atomic);
2195
2196void sdhci_set_bus_width(struct sdhci_host *host, int width)
2197{
2198        u8 ctrl;
2199
2200        ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
2201        if (width == MMC_BUS_WIDTH_8) {
2202                ctrl &= ~SDHCI_CTRL_4BITBUS;
2203                ctrl |= SDHCI_CTRL_8BITBUS;
2204        } else {
2205                if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
2206                        ctrl &= ~SDHCI_CTRL_8BITBUS;
2207                if (width == MMC_BUS_WIDTH_4)
2208                        ctrl |= SDHCI_CTRL_4BITBUS;
2209                else
2210                        ctrl &= ~SDHCI_CTRL_4BITBUS;
2211        }
2212        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
2213}
2214EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
2215
2216void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
2217{
2218        u16 ctrl_2;
2219
2220        ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2221        /* Select Bus Speed Mode for host */
2222        ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
2223        if ((timing == MMC_TIMING_MMC_HS200) ||
2224            (timing == MMC_TIMING_UHS_SDR104))
2225                ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
2226        else if (timing == MMC_TIMING_UHS_SDR12)
2227                ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
2228        else if (timing == MMC_TIMING_UHS_SDR25)
2229                ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
2230        else if (timing == MMC_TIMING_UHS_SDR50)
2231                ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
2232        else if ((timing == MMC_TIMING_UHS_DDR50) ||
2233                 (timing == MMC_TIMING_MMC_DDR52))
2234                ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
2235        else if (timing == MMC_TIMING_MMC_HS400)
2236                ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
2237        sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
2238}
2239EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
2240
2241void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
2242{
2243        struct sdhci_host *host = mmc_priv(mmc);
2244        u8 ctrl;
2245
2246        if (ios->power_mode == MMC_POWER_UNDEFINED)
2247                return;
2248
2249        if (host->flags & SDHCI_DEVICE_DEAD) {
2250                if (!IS_ERR(mmc->supply.vmmc) &&
2251                    ios->power_mode == MMC_POWER_OFF)
2252                        mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
2253                return;
2254        }
2255
2256        /*
2257         * Reset the chip on each power off.
2258         * Should clear out any weird states.
2259         */
2260        if (ios->power_mode == MMC_POWER_OFF) {
2261                sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
2262                sdhci_reinit(host);
2263        }
2264
2265        if (host->version >= SDHCI_SPEC_300 &&
2266                (ios->power_mode == MMC_POWER_UP) &&
2267                !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
2268                sdhci_enable_preset_value(host, false);
2269
2270        if (!ios->clock || ios->clock != host->clock) {
2271                host->ops->set_clock(host, ios->clock);
2272                host->clock = ios->clock;
2273
2274                if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
2275                    host->clock) {
2276                        host->timeout_clk = mmc->actual_clock ?
2277                                                mmc->actual_clock / 1000 :
2278                                                host->clock / 1000;
2279                        mmc->max_busy_timeout =
2280                                host->ops->get_max_timeout_count ?
2281                                host->ops->get_max_timeout_count(host) :
2282                                1 << 27;
2283                        mmc->max_busy_timeout /= host->timeout_clk;
2284                }
2285        }
2286
2287        if (host->ops->set_power)
2288                host->ops->set_power(host, ios->power_mode, ios->vdd);
2289        else
2290                sdhci_set_power(host, ios->power_mode, ios->vdd);
2291
2292        if (host->ops->platform_send_init_74_clocks)
2293                host->ops->platform_send_init_74_clocks(host, ios->power_mode);
2294
2295        host->ops->set_bus_width(host, ios->bus_width);
2296
2297        ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
2298
2299        if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
2300                if (ios->timing == MMC_TIMING_SD_HS ||
2301                     ios->timing == MMC_TIMING_MMC_HS ||
2302                     ios->timing == MMC_TIMING_MMC_HS400 ||
2303                     ios->timing == MMC_TIMING_MMC_HS200 ||
2304                     ios->timing == MMC_TIMING_MMC_DDR52 ||
2305                     ios->timing == MMC_TIMING_UHS_SDR50 ||
2306                     ios->timing == MMC_TIMING_UHS_SDR104 ||
2307                     ios->timing == MMC_TIMING_UHS_DDR50 ||
2308                     ios->timing == MMC_TIMING_UHS_SDR25)
2309                        ctrl |= SDHCI_CTRL_HISPD;
2310                else
2311                        ctrl &= ~SDHCI_CTRL_HISPD;
2312        }
2313
2314        if (host->version >= SDHCI_SPEC_300) {
2315                u16 clk, ctrl_2;
2316
2317                if (!host->preset_enabled) {
2318                        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
2319                        /*
2320                         * We only need to set Driver Strength if the
2321                         * preset value enable is not set.
2322                         */
2323                        ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2324                        ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
2325                        if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
2326                                ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
2327                        else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
2328                                ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
2329                        else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
2330                                ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
2331                        else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
2332                                ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
2333                        else {
2334                                pr_warn("%s: invalid driver type, default to driver type B\n",
2335                                        mmc_hostname(mmc));
2336                                ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
2337                        }
2338
2339                        sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
2340                } else {
2341                        /*
2342                         * According to SDHC Spec v3.00, if the Preset Value
2343                         * Enable in the Host Control 2 register is set, we
2344                         * need to reset SD Clock Enable before changing High
2345                         * Speed Enable to avoid generating clock gliches.
2346                         */
2347
2348                        /* Reset SD Clock Enable */
2349                        clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
2350                        clk &= ~SDHCI_CLOCK_CARD_EN;
2351                        sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
2352
2353                        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
2354
2355                        /* Re-enable SD Clock */
2356                        host->ops->set_clock(host, host->clock);
2357                }
2358
2359                /* Reset SD Clock Enable */
2360                clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
2361                clk &= ~SDHCI_CLOCK_CARD_EN;
2362                sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
2363
2364                host->ops->set_uhs_signaling(host, ios->timing);
2365                host->timing = ios->timing;
2366
2367                if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
2368                                ((ios->timing == MMC_TIMING_UHS_SDR12) ||
2369                                 (ios->timing == MMC_TIMING_UHS_SDR25) ||
2370                                 (ios->timing == MMC_TIMING_UHS_SDR50) ||
2371                                 (ios->timing == MMC_TIMING_UHS_SDR104) ||
2372                                 (ios->timing == MMC_TIMING_UHS_DDR50) ||
2373                                 (ios->timing == MMC_TIMING_MMC_DDR52))) {
2374                        u16 preset;
2375
2376                        sdhci_enable_preset_value(host, true);
2377                        preset = sdhci_get_preset_value(host);
2378                        ios->drv_type = FIELD_GET(SDHCI_PRESET_DRV_MASK,
2379                                                  preset);
2380                }
2381
2382                /* Re-enable SD Clock */
2383                host->ops->set_clock(host, host->clock);
2384        } else
2385                sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
2386
2387        /*
2388         * Some (ENE) controllers go apeshit on some ios operation,
2389         * signalling timeout and CRC errors even on CMD0. Resetting
2390         * it on each ios seems to solve the problem.
2391         */
2392        if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
2393                sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
2394}
2395EXPORT_SYMBOL_GPL(sdhci_set_ios);
2396
2397static int sdhci_get_cd(struct mmc_host *mmc)
2398{
2399        struct sdhci_host *host = mmc_priv(mmc);
2400        int gpio_cd = mmc_gpio_get_cd(mmc);
2401
2402        if (host->flags & SDHCI_DEVICE_DEAD)
2403                return 0;
2404
2405        /* If nonremovable, assume that the card is always present. */
2406        if (!mmc_card_is_removable(mmc))
2407                return 1;
2408
2409        /*
2410         * Try slot gpio detect, if defined it take precedence
2411         * over build in controller functionality
2412         */
2413        if (gpio_cd >= 0)
2414                return !!gpio_cd;
2415
2416        /* If polling, assume that the card is always present. */
2417        if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
2418                return 1;
2419
2420        /* Host native card detect */
2421        return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
2422}
2423
2424static int sdhci_check_ro(struct sdhci_host *host)
2425{
2426        unsigned long flags;
2427        int is_readonly;
2428
2429        spin_lock_irqsave(&host->lock, flags);
2430
2431        if (host->flags & SDHCI_DEVICE_DEAD)
2432                is_readonly = 0;
2433        else if (host->ops->get_ro)
2434                is_readonly = host->ops->get_ro(host);
2435        else if (mmc_can_gpio_ro(host->mmc))
2436                is_readonly = mmc_gpio_get_ro(host->mmc);
2437        else
2438                is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
2439                                & SDHCI_WRITE_PROTECT);
2440
2441        spin_unlock_irqrestore(&host->lock, flags);
2442
2443        /* This quirk needs to be replaced by a callback-function later */
2444        return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
2445                !is_readonly : is_readonly;
2446}
2447
2448#define SAMPLE_COUNT    5
2449
2450static int sdhci_get_ro(struct mmc_host *mmc)
2451{
2452        struct sdhci_host *host = mmc_priv(mmc);
2453        int i, ro_count;
2454
2455        if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
2456                return sdhci_check_ro(host);
2457
2458        ro_count = 0;
2459        for (i = 0; i < SAMPLE_COUNT; i++) {
2460                if (sdhci_check_ro(host)) {
2461                        if (++ro_count > SAMPLE_COUNT / 2)
2462                                return 1;
2463                }
2464                msleep(30);
2465        }
2466        return 0;
2467}
2468
2469static void sdhci_hw_reset(struct mmc_host *mmc)
2470{
2471        struct sdhci_host *host = mmc_priv(mmc);
2472
2473        if (host->ops && host->ops->hw_reset)
2474                host->ops->hw_reset(host);
2475}
2476
2477static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
2478{
2479        if (!(host->flags & SDHCI_DEVICE_DEAD)) {
2480                if (enable)
2481                        host->ier |= SDHCI_INT_CARD_INT;
2482                else
2483                        host->ier &= ~SDHCI_INT_CARD_INT;
2484
2485                sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2486                sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2487        }
2488}
2489
2490void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
2491{
2492        struct sdhci_host *host = mmc_priv(mmc);
2493        unsigned long flags;
2494
2495        if (enable)
2496                pm_runtime_get_noresume(mmc_dev(mmc));
2497
2498        spin_lock_irqsave(&host->lock, flags);
2499        sdhci_enable_sdio_irq_nolock(host, enable);
2500        spin_unlock_irqrestore(&host->lock, flags);
2501
2502        if (!enable)
2503                pm_runtime_put_noidle(mmc_dev(mmc));
2504}
2505EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
2506
2507static void sdhci_ack_sdio_irq(struct mmc_host *mmc)
2508{
2509        struct sdhci_host *host = mmc_priv(mmc);
2510        unsigned long flags;
2511
2512        spin_lock_irqsave(&host->lock, flags);
2513        sdhci_enable_sdio_irq_nolock(host, true);
2514        spin_unlock_irqrestore(&host->lock, flags);
2515}
2516
2517int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
2518                                      struct mmc_ios *ios)
2519{
2520        struct sdhci_host *host = mmc_priv(mmc);
2521        u16 ctrl;
2522        int ret;
2523
2524        /*
2525         * Signal Voltage Switching is only applicable for Host Controllers
2526         * v3.00 and above.
2527         */
2528        if (host->version < SDHCI_SPEC_300)
2529                return 0;
2530
2531        ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2532
2533        switch (ios->signal_voltage) {
2534        case MMC_SIGNAL_VOLTAGE_330:
2535                if (!(host->flags & SDHCI_SIGNALING_330))
2536                        return -EINVAL;
2537                /* Set 1.8V Signal Enable in the Host Control2 register to 0 */
2538                ctrl &= ~SDHCI_CTRL_VDD_180;
2539                sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2540
2541                if (!IS_ERR(mmc->supply.vqmmc)) {
2542                        ret = mmc_regulator_set_vqmmc(mmc, ios);
2543                        if (ret < 0) {
2544                                pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
2545                                        mmc_hostname(mmc));
2546                                return -EIO;
2547                        }
2548                }
2549                /* Wait for 5ms */
2550                usleep_range(5000, 5500);
2551
2552                /* 3.3V regulator output should be stable within 5 ms */
2553                ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2554                if (!(ctrl & SDHCI_CTRL_VDD_180))
2555                        return 0;
2556
2557                pr_warn("%s: 3.3V regulator output did not become stable\n",
2558                        mmc_hostname(mmc));
2559
2560                return -EAGAIN;
2561        case MMC_SIGNAL_VOLTAGE_180:
2562                if (!(host->flags & SDHCI_SIGNALING_180))
2563                        return -EINVAL;
2564                if (!IS_ERR(mmc->supply.vqmmc)) {
2565                        ret = mmc_regulator_set_vqmmc(mmc, ios);
2566                        if (ret < 0) {
2567                                pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
2568                                        mmc_hostname(mmc));
2569                                return -EIO;
2570                        }
2571                }
2572
2573                /*
2574                 * Enable 1.8V Signal Enable in the Host Control2
2575                 * register
2576                 */
2577                ctrl |= SDHCI_CTRL_VDD_180;
2578                sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2579
2580                /* Some controller need to do more when switching */
2581                if (host->ops->voltage_switch)
2582                        host->ops->voltage_switch(host);
2583
2584                /* 1.8V regulator output should be stable within 5 ms */
2585                ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2586                if (ctrl & SDHCI_CTRL_VDD_180)
2587                        return 0;
2588
2589                pr_warn("%s: 1.8V regulator output did not become stable\n",
2590                        mmc_hostname(mmc));
2591
2592                return -EAGAIN;
2593        case MMC_SIGNAL_VOLTAGE_120:
2594                if (!(host->flags & SDHCI_SIGNALING_120))
2595                        return -EINVAL;
2596                if (!IS_ERR(mmc->supply.vqmmc)) {
2597                        ret = mmc_regulator_set_vqmmc(mmc, ios);
2598                        if (ret < 0) {
2599                                pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2600                                        mmc_hostname(mmc));
2601                                return -EIO;
2602                        }
2603                }
2604                return 0;
2605        default:
2606                /* No signal voltage switch required */
2607                return 0;
2608        }
2609}
2610EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2611
2612static int sdhci_card_busy(struct mmc_host *mmc)
2613{
2614        struct sdhci_host *host = mmc_priv(mmc);
2615        u32 present_state;
2616
2617        /* Check whether DAT[0] is 0 */
2618        present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2619
2620        return !(present_state & SDHCI_DATA_0_LVL_MASK);
2621}
2622
2623static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2624{
2625        struct sdhci_host *host = mmc_priv(mmc);
2626        unsigned long flags;
2627
2628        spin_lock_irqsave(&host->lock, flags);
2629        host->flags |= SDHCI_HS400_TUNING;
2630        spin_unlock_irqrestore(&host->lock, flags);
2631
2632        return 0;
2633}
2634
2635void sdhci_start_tuning(struct sdhci_host *host)
2636{
2637        u16 ctrl;
2638
2639        ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2640        ctrl |= SDHCI_CTRL_EXEC_TUNING;
2641        if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2642                ctrl |= SDHCI_CTRL_TUNED_CLK;
2643        sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2644
2645        /*
2646         * As per the Host Controller spec v3.00, tuning command
2647         * generates Buffer Read Ready interrupt, so enable that.
2648         *
2649         * Note: The spec clearly says that when tuning sequence
2650         * is being performed, the controller does not generate
2651         * interrupts other than Buffer Read Ready interrupt. But
2652         * to make sure we don't hit a controller bug, we _only_
2653         * enable Buffer Read Ready interrupt here.
2654         */
2655        sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2656        sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2657}
2658EXPORT_SYMBOL_GPL(sdhci_start_tuning);
2659
2660void sdhci_end_tuning(struct sdhci_host *host)
2661{
2662        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2663        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2664}
2665EXPORT_SYMBOL_GPL(sdhci_end_tuning);
2666
2667void sdhci_reset_tuning(struct sdhci_host *host)
2668{
2669        u16 ctrl;
2670
2671        ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2672        ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2673        ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2674        sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2675}
2676EXPORT_SYMBOL_GPL(sdhci_reset_tuning);
2677
2678void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2679{
2680        sdhci_reset_tuning(host);
2681
2682        sdhci_do_reset(host, SDHCI_RESET_CMD);
2683        sdhci_do_reset(host, SDHCI_RESET_DATA);
2684
2685        sdhci_end_tuning(host);
2686
2687        mmc_send_abort_tuning(host->mmc, opcode);
2688}
2689EXPORT_SYMBOL_GPL(sdhci_abort_tuning);
2690
2691/*
2692 * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2693 * tuning command does not have a data payload (or rather the hardware does it
2694 * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2695 * interrupt setup is different to other commands and there is no timeout
2696 * interrupt so special handling is needed.
2697 */
2698void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2699{
2700        struct mmc_host *mmc = host->mmc;
2701        struct mmc_command cmd = {};
2702        struct mmc_request mrq = {};
2703        unsigned long flags;
2704        u32 b = host->sdma_boundary;
2705
2706        spin_lock_irqsave(&host->lock, flags);
2707
2708        cmd.opcode = opcode;
2709        cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2710        cmd.mrq = &mrq;
2711
2712        mrq.cmd = &cmd;
2713        /*
2714         * In response to CMD19, the card sends 64 bytes of tuning
2715         * block to the Host Controller. So we set the block size
2716         * to 64 here.
2717         */
2718        if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2719            mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2720                sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2721        else
2722                sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2723
2724        /*
2725         * The tuning block is sent by the card to the host controller.
2726         * So we set the TRNS_READ bit in the Transfer Mode register.
2727         * This also takes care of setting DMA Enable and Multi Block
2728         * Select in the same register to 0.
2729         */
2730        sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2731
2732        if (!sdhci_send_command_retry(host, &cmd, flags)) {
2733                spin_unlock_irqrestore(&host->lock, flags);
2734                host->tuning_done = 0;
2735                return;
2736        }
2737
2738        host->cmd = NULL;
2739
2740        sdhci_del_timer(host, &mrq);
2741
2742        host->tuning_done = 0;
2743
2744        spin_unlock_irqrestore(&host->lock, flags);
2745
2746        /* Wait for Buffer Read Ready interrupt */
2747        wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2748                           msecs_to_jiffies(50));
2749
2750}
2751EXPORT_SYMBOL_GPL(sdhci_send_tuning);
2752
2753static int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2754{
2755        int i;
2756
2757        /*
2758         * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2759         * of loops reaches tuning loop count.
2760         */
2761        for (i = 0; i < host->tuning_loop_count; i++) {
2762                u16 ctrl;
2763
2764                sdhci_send_tuning(host, opcode);
2765
2766                if (!host->tuning_done) {
2767                        pr_debug("%s: Tuning timeout, falling back to fixed sampling clock\n",
2768                                 mmc_hostname(host->mmc));
2769                        sdhci_abort_tuning(host, opcode);
2770                        return -ETIMEDOUT;
2771                }
2772
2773                /* Spec does not require a delay between tuning cycles */
2774                if (host->tuning_delay > 0)
2775                        mdelay(host->tuning_delay);
2776
2777                ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2778                if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2779                        if (ctrl & SDHCI_CTRL_TUNED_CLK)
2780                                return 0; /* Success! */
2781                        break;
2782                }
2783
2784        }
2785
2786        pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2787                mmc_hostname(host->mmc));
2788        sdhci_reset_tuning(host);
2789        return -EAGAIN;
2790}
2791
2792int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2793{
2794        struct sdhci_host *host = mmc_priv(mmc);
2795        int err = 0;
2796        unsigned int tuning_count = 0;
2797        bool hs400_tuning;
2798
2799        hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2800
2801        if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2802                tuning_count = host->tuning_count;
2803
2804        /*
2805         * The Host Controller needs tuning in case of SDR104 and DDR50
2806         * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2807         * the Capabilities register.
2808         * If the Host Controller supports the HS200 mode then the
2809         * tuning function has to be executed.
2810         */
2811        switch (host->timing) {
2812        /* HS400 tuning is done in HS200 mode */
2813        case MMC_TIMING_MMC_HS400:
2814                err = -EINVAL;
2815                goto out;
2816
2817        case MMC_TIMING_MMC_HS200:
2818                /*
2819                 * Periodic re-tuning for HS400 is not expected to be needed, so
2820                 * disable it here.
2821                 */
2822                if (hs400_tuning)
2823                        tuning_count = 0;
2824                break;
2825
2826        case MMC_TIMING_UHS_SDR104:
2827        case MMC_TIMING_UHS_DDR50:
2828                break;
2829
2830        case MMC_TIMING_UHS_SDR50:
2831                if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2832                        break;
2833                fallthrough;
2834
2835        default:
2836                goto out;
2837        }
2838
2839        if (host->ops->platform_execute_tuning) {
2840                err = host->ops->platform_execute_tuning(host, opcode);
2841                goto out;
2842        }
2843
2844        mmc->retune_period = tuning_count;
2845
2846        if (host->tuning_delay < 0)
2847                host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2848
2849        sdhci_start_tuning(host);
2850
2851        host->tuning_err = __sdhci_execute_tuning(host, opcode);
2852
2853        sdhci_end_tuning(host);
2854out:
2855        host->flags &= ~SDHCI_HS400_TUNING;
2856
2857        return err;
2858}
2859EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2860
2861static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2862{
2863        /* Host Controller v3.00 defines preset value registers */
2864        if (host->version < SDHCI_SPEC_300)
2865                return;
2866
2867        /*
2868         * We only enable or disable Preset Value if they are not already
2869         * enabled or disabled respectively. Otherwise, we bail out.
2870         */
2871        if (host->preset_enabled != enable) {
2872                u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2873
2874                if (enable)
2875                        ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2876                else
2877                        ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2878
2879                sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2880
2881                if (enable)
2882                        host->flags |= SDHCI_PV_ENABLED;
2883                else
2884                        host->flags &= ~SDHCI_PV_ENABLED;
2885
2886                host->preset_enabled = enable;
2887        }
2888}
2889
2890static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2891                                int err)
2892{
2893        struct mmc_data *data = mrq->data;
2894
2895        if (data->host_cookie != COOKIE_UNMAPPED)
2896                dma_unmap_sg(mmc_dev(mmc), data->sg, data->sg_len,
2897                             mmc_get_dma_dir(data));
2898
2899        data->host_cookie = COOKIE_UNMAPPED;
2900}
2901
2902static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2903{
2904        struct sdhci_host *host = mmc_priv(mmc);
2905
2906        mrq->data->host_cookie = COOKIE_UNMAPPED;
2907
2908        /*
2909         * No pre-mapping in the pre hook if we're using the bounce buffer,
2910         * for that we would need two bounce buffers since one buffer is
2911         * in flight when this is getting called.
2912         */
2913        if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2914                sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2915}
2916
2917static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2918{
2919        if (host->data_cmd) {
2920                host->data_cmd->error = err;
2921                sdhci_finish_mrq(host, host->data_cmd->mrq);
2922        }
2923
2924        if (host->cmd) {
2925                host->cmd->error = err;
2926                sdhci_finish_mrq(host, host->cmd->mrq);
2927        }
2928}
2929
2930static void sdhci_card_event(struct mmc_host *mmc)
2931{
2932        struct sdhci_host *host = mmc_priv(mmc);
2933        unsigned long flags;
2934        int present;
2935
2936        /* First check if client has provided their own card event */
2937        if (host->ops->card_event)
2938                host->ops->card_event(host);
2939
2940        present = mmc->ops->get_cd(mmc);
2941
2942        spin_lock_irqsave(&host->lock, flags);
2943
2944        /* Check sdhci_has_requests() first in case we are runtime suspended */
2945        if (sdhci_has_requests(host) && !present) {
2946                pr_err("%s: Card removed during transfer!\n",
2947                        mmc_hostname(mmc));
2948                pr_err("%s: Resetting controller.\n",
2949                        mmc_hostname(mmc));
2950
2951                sdhci_do_reset(host, SDHCI_RESET_CMD);
2952                sdhci_do_reset(host, SDHCI_RESET_DATA);
2953
2954                sdhci_error_out_mrqs(host, -ENOMEDIUM);
2955        }
2956
2957        spin_unlock_irqrestore(&host->lock, flags);
2958}
2959
2960static const struct mmc_host_ops sdhci_ops = {
2961        .request        = sdhci_request,
2962        .post_req       = sdhci_post_req,
2963        .pre_req        = sdhci_pre_req,
2964        .set_ios        = sdhci_set_ios,
2965        .get_cd         = sdhci_get_cd,
2966        .get_ro         = sdhci_get_ro,
2967        .hw_reset       = sdhci_hw_reset,
2968        .enable_sdio_irq = sdhci_enable_sdio_irq,
2969        .ack_sdio_irq    = sdhci_ack_sdio_irq,
2970        .start_signal_voltage_switch    = sdhci_start_signal_voltage_switch,
2971        .prepare_hs400_tuning           = sdhci_prepare_hs400_tuning,
2972        .execute_tuning                 = sdhci_execute_tuning,
2973        .card_event                     = sdhci_card_event,
2974        .card_busy      = sdhci_card_busy,
2975};
2976
2977/*****************************************************************************\
2978 *                                                                           *
2979 * Request done                                                              *
2980 *                                                                           *
2981\*****************************************************************************/
2982
2983static bool sdhci_request_done(struct sdhci_host *host)
2984{
2985        unsigned long flags;
2986        struct mmc_request *mrq;
2987        int i;
2988
2989        spin_lock_irqsave(&host->lock, flags);
2990
2991        for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2992                mrq = host->mrqs_done[i];
2993                if (mrq)
2994                        break;
2995        }
2996
2997        if (!mrq) {
2998                spin_unlock_irqrestore(&host->lock, flags);
2999                return true;
3000        }
3001
3002        /*
3003         * The controller needs a reset of internal state machines
3004         * upon error conditions.
3005         */
3006        if (sdhci_needs_reset(host, mrq)) {
3007                /*
3008                 * Do not finish until command and data lines are available for
3009                 * reset. Note there can only be one other mrq, so it cannot
3010                 * also be in mrqs_done, otherwise host->cmd and host->data_cmd
3011                 * would both be null.
3012                 */
3013                if (host->cmd || host->data_cmd) {
3014                        spin_unlock_irqrestore(&host->lock, flags);
3015                        return true;
3016                }
3017
3018                /* Some controllers need this kick or reset won't work here */
3019                if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
3020                        /* This is to force an update */
3021                        host->ops->set_clock(host, host->clock);
3022
3023                /*
3024                 * Spec says we should do both at the same time, but Ricoh
3025                 * controllers do not like that.
3026                 */
3027                sdhci_do_reset(host, SDHCI_RESET_CMD);
3028                sdhci_do_reset(host, SDHCI_RESET_DATA);
3029
3030                host->pending_reset = false;
3031        }
3032
3033        /*
3034         * Always unmap the data buffers if they were mapped by
3035         * sdhci_prepare_data() whenever we finish with a request.
3036         * This avoids leaking DMA mappings on error.
3037         */
3038        if (host->flags & SDHCI_REQ_USE_DMA) {
3039                struct mmc_data *data = mrq->data;
3040
3041                if (host->use_external_dma && data &&
3042                    (mrq->cmd->error || data->error)) {
3043                        struct dma_chan *chan = sdhci_external_dma_channel(host, data);
3044
3045                        host->mrqs_done[i] = NULL;
3046                        spin_unlock_irqrestore(&host->lock, flags);
3047                        dmaengine_terminate_sync(chan);
3048                        spin_lock_irqsave(&host->lock, flags);
3049                        sdhci_set_mrq_done(host, mrq);
3050                }
3051
3052                if (data && data->host_cookie == COOKIE_MAPPED) {
3053                        if (host->bounce_buffer) {
3054                                /*
3055                                 * On reads, copy the bounced data into the
3056                                 * sglist
3057                                 */
3058                                if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
3059                                        unsigned int length = data->bytes_xfered;
3060
3061                                        if (length > host->bounce_buffer_size) {
3062                                                pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
3063                                                       mmc_hostname(host->mmc),
3064                                                       host->bounce_buffer_size,
3065                                                       data->bytes_xfered);
3066                                                /* Cap it down and continue */
3067                                                length = host->bounce_buffer_size;
3068                                        }
3069                                        dma_sync_single_for_cpu(
3070                                                mmc_dev(host->mmc),
3071                                                host->bounce_addr,
3072                                                host->bounce_buffer_size,
3073                                                DMA_FROM_DEVICE);
3074                                        sg_copy_from_buffer(data->sg,
3075                                                data->sg_len,
3076                                                host->bounce_buffer,
3077                                                length);
3078                                } else {
3079                                        /* No copying, just switch ownership */
3080                                        dma_sync_single_for_cpu(
3081                                                mmc_dev(host->mmc),
3082                                                host->bounce_addr,
3083                                                host->bounce_buffer_size,
3084                                                mmc_get_dma_dir(data));
3085                                }
3086                        } else {
3087                                /* Unmap the raw data */
3088                                dma_unmap_sg(mmc_dev(host->mmc), data->sg,
3089                                             data->sg_len,
3090                                             mmc_get_dma_dir(data));
3091                        }
3092                        data->host_cookie = COOKIE_UNMAPPED;
3093                }
3094        }
3095
3096        host->mrqs_done[i] = NULL;
3097
3098        spin_unlock_irqrestore(&host->lock, flags);
3099
3100        if (host->ops->request_done)
3101                host->ops->request_done(host, mrq);
3102        else
3103                mmc_request_done(host->mmc, mrq);
3104
3105        return false;
3106}
3107
3108static void sdhci_complete_work(struct work_struct *work)
3109{
3110        struct sdhci_host *host = container_of(work, struct sdhci_host,
3111                                               complete_work);
3112
3113        while (!sdhci_request_done(host))
3114                ;
3115}
3116
3117static void sdhci_timeout_timer(struct timer_list *t)
3118{
3119        struct sdhci_host *host;
3120        unsigned long flags;
3121
3122        host = from_timer(host, t, timer);
3123
3124        spin_lock_irqsave(&host->lock, flags);
3125
3126        if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
3127                pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
3128                       mmc_hostname(host->mmc));
3129                sdhci_dumpregs(host);
3130
3131                host->cmd->error = -ETIMEDOUT;
3132                sdhci_finish_mrq(host, host->cmd->mrq);
3133        }
3134
3135        spin_unlock_irqrestore(&host->lock, flags);
3136}
3137
3138static void sdhci_timeout_data_timer(struct timer_list *t)
3139{
3140        struct sdhci_host *host;
3141        unsigned long flags;
3142
3143        host = from_timer(host, t, data_timer);
3144
3145        spin_lock_irqsave(&host->lock, flags);
3146
3147        if (host->data || host->data_cmd ||
3148            (host->cmd && sdhci_data_line_cmd(host->cmd))) {
3149                pr_err("%s: Timeout waiting for hardware interrupt.\n",
3150                       mmc_hostname(host->mmc));
3151                sdhci_dumpregs(host);
3152
3153                if (host->data) {
3154                        host->data->error = -ETIMEDOUT;
3155                        __sdhci_finish_data(host, true);
3156                        queue_work(host->complete_wq, &host->complete_work);
3157                } else if (host->data_cmd) {
3158                        host->data_cmd->error = -ETIMEDOUT;
3159                        sdhci_finish_mrq(host, host->data_cmd->mrq);
3160                } else {
3161                        host->cmd->error = -ETIMEDOUT;
3162                        sdhci_finish_mrq(host, host->cmd->mrq);
3163                }
3164        }
3165
3166        spin_unlock_irqrestore(&host->lock, flags);
3167}
3168
3169/*****************************************************************************\
3170 *                                                                           *
3171 * Interrupt handling                                                        *
3172 *                                                                           *
3173\*****************************************************************************/
3174
3175static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p)
3176{
3177        /* Handle auto-CMD12 error */
3178        if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) {
3179                struct mmc_request *mrq = host->data_cmd->mrq;
3180                u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
3181                int data_err_bit = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
3182                                   SDHCI_INT_DATA_TIMEOUT :
3183                                   SDHCI_INT_DATA_CRC;
3184
3185                /* Treat auto-CMD12 error the same as data error */
3186                if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) {
3187                        *intmask_p |= data_err_bit;
3188                        return;
3189                }
3190        }
3191
3192        if (!host->cmd) {
3193                /*
3194                 * SDHCI recovers from errors by resetting the cmd and data
3195                 * circuits.  Until that is done, there very well might be more
3196                 * interrupts, so ignore them in that case.
3197                 */
3198                if (host->pending_reset)
3199                        return;
3200                pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
3201                       mmc_hostname(host->mmc), (unsigned)intmask);
3202                sdhci_dumpregs(host);
3203                return;
3204        }
3205
3206        if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
3207                       SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
3208                if (intmask & SDHCI_INT_TIMEOUT)
3209                        host->cmd->error = -ETIMEDOUT;
3210                else
3211                        host->cmd->error = -EILSEQ;
3212
3213                /* Treat data command CRC error the same as data CRC error */
3214                if (host->cmd->data &&
3215                    (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
3216                     SDHCI_INT_CRC) {
3217                        host->cmd = NULL;
3218                        *intmask_p |= SDHCI_INT_DATA_CRC;
3219                        return;
3220                }
3221
3222                __sdhci_finish_mrq(host, host->cmd->mrq);
3223                return;
3224        }
3225
3226        /* Handle auto-CMD23 error */
3227        if (intmask & SDHCI_INT_AUTO_CMD_ERR) {
3228                struct mmc_request *mrq = host->cmd->mrq;
3229                u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
3230                int err = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
3231                          -ETIMEDOUT :
3232                          -EILSEQ;
3233
3234                if (mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
3235                        mrq->sbc->error = err;
3236                        __sdhci_finish_mrq(host, mrq);
3237                        return;
3238                }
3239        }
3240
3241        if (intmask & SDHCI_INT_RESPONSE)
3242                sdhci_finish_command(host);
3243}
3244
3245static void sdhci_adma_show_error(struct sdhci_host *host)
3246{
3247        void *desc = host->adma_table;
3248        dma_addr_t dma = host->adma_addr;
3249
3250        sdhci_dumpregs(host);
3251
3252        while (true) {
3253                struct sdhci_adma2_64_desc *dma_desc = desc;
3254
3255                if (host->flags & SDHCI_USE_64_BIT_DMA)
3256                        SDHCI_DUMP("%08llx: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
3257                            (unsigned long long)dma,
3258                            le32_to_cpu(dma_desc->addr_hi),
3259                            le32_to_cpu(dma_desc->addr_lo),
3260                            le16_to_cpu(dma_desc->len),
3261                            le16_to_cpu(dma_desc->cmd));
3262                else
3263                        SDHCI_DUMP("%08llx: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
3264                            (unsigned long long)dma,
3265                            le32_to_cpu(dma_desc->addr_lo),
3266                            le16_to_cpu(dma_desc->len),
3267                            le16_to_cpu(dma_desc->cmd));
3268
3269                desc += host->desc_sz;
3270                dma += host->desc_sz;
3271
3272                if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
3273                        break;
3274        }
3275}
3276
3277static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
3278{
3279        u32 command;
3280
3281        /* CMD19 generates _only_ Buffer Read Ready interrupt */
3282        if (intmask & SDHCI_INT_DATA_AVAIL) {
3283                command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
3284                if (command == MMC_SEND_TUNING_BLOCK ||
3285                    command == MMC_SEND_TUNING_BLOCK_HS200) {
3286                        host->tuning_done = 1;
3287                        wake_up(&host->buf_ready_int);
3288                        return;
3289                }
3290        }
3291
3292        if (!host->data) {
3293                struct mmc_command *data_cmd = host->data_cmd;
3294
3295                /*
3296                 * The "data complete" interrupt is also used to
3297                 * indicate that a busy state has ended. See comment
3298                 * above in sdhci_cmd_irq().
3299                 */
3300                if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
3301                        if (intmask & SDHCI_INT_DATA_TIMEOUT) {
3302                                host->data_cmd = NULL;
3303                                data_cmd->error = -ETIMEDOUT;
3304                                __sdhci_finish_mrq(host, data_cmd->mrq);
3305                                return;
3306                        }
3307                        if (intmask & SDHCI_INT_DATA_END) {
3308                                host->data_cmd = NULL;
3309                                /*
3310                                 * Some cards handle busy-end interrupt
3311                                 * before the command completed, so make
3312                                 * sure we do things in the proper order.
3313                                 */
3314                                if (host->cmd == data_cmd)
3315                                        return;
3316
3317                                __sdhci_finish_mrq(host, data_cmd->mrq);
3318                                return;
3319                        }
3320                }
3321
3322                /*
3323                 * SDHCI recovers from errors by resetting the cmd and data
3324                 * circuits. Until that is done, there very well might be more
3325                 * interrupts, so ignore them in that case.
3326                 */
3327                if (host->pending_reset)
3328                        return;
3329
3330                pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
3331                       mmc_hostname(host->mmc), (unsigned)intmask);
3332                sdhci_dumpregs(host);
3333
3334                return;
3335        }
3336
3337        if (intmask & SDHCI_INT_DATA_TIMEOUT)
3338                host->data->error = -ETIMEDOUT;
3339        else if (intmask & SDHCI_INT_DATA_END_BIT)
3340                host->data->error = -EILSEQ;
3341        else if ((intmask & SDHCI_INT_DATA_CRC) &&
3342                SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
3343                        != MMC_BUS_TEST_R)
3344                host->data->error = -EILSEQ;
3345        else if (intmask & SDHCI_INT_ADMA_ERROR) {
3346                pr_err("%s: ADMA error: 0x%08x\n", mmc_hostname(host->mmc),
3347                       intmask);
3348                sdhci_adma_show_error(host);
3349                host->data->error = -EIO;
3350                if (host->ops->adma_workaround)
3351                        host->ops->adma_workaround(host, intmask);
3352        }
3353
3354        if (host->data->error)
3355                sdhci_finish_data(host);
3356        else {
3357                if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
3358                        sdhci_transfer_pio(host);
3359
3360                /*
3361                 * We currently don't do anything fancy with DMA
3362                 * boundaries, but as we can't disable the feature
3363                 * we need to at least restart the transfer.
3364                 *
3365                 * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
3366                 * should return a valid address to continue from, but as
3367                 * some controllers are faulty, don't trust them.
3368                 */
3369                if (intmask & SDHCI_INT_DMA_END) {
3370                        dma_addr_t dmastart, dmanow;
3371
3372                        dmastart = sdhci_sdma_address(host);
3373                        dmanow = dmastart + host->data->bytes_xfered;
3374                        /*
3375                         * Force update to the next DMA block boundary.
3376                         */
3377                        dmanow = (dmanow &
3378                                ~((dma_addr_t)SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
3379                                SDHCI_DEFAULT_BOUNDARY_SIZE;
3380                        host->data->bytes_xfered = dmanow - dmastart;
3381                        DBG("DMA base %pad, transferred 0x%06x bytes, next %pad\n",
3382                            &dmastart, host->data->bytes_xfered, &dmanow);
3383                        sdhci_set_sdma_addr(host, dmanow);
3384                }
3385
3386                if (intmask & SDHCI_INT_DATA_END) {
3387                        if (host->cmd == host->data_cmd) {
3388                                /*
3389                                 * Data managed to finish before the
3390                                 * command completed. Make sure we do
3391                                 * things in the proper order.
3392                                 */
3393                                host->data_early = 1;
3394                        } else {
3395                                sdhci_finish_data(host);
3396                        }
3397                }
3398        }
3399}
3400
3401static inline bool sdhci_defer_done(struct sdhci_host *host,
3402                                    struct mmc_request *mrq)
3403{
3404        struct mmc_data *data = mrq->data;
3405
3406        return host->pending_reset || host->always_defer_done ||
3407               ((host->flags & SDHCI_REQ_USE_DMA) && data &&
3408                data->host_cookie == COOKIE_MAPPED);
3409}
3410
3411static irqreturn_t sdhci_irq(int irq, void *dev_id)
3412{
3413        struct mmc_request *mrqs_done[SDHCI_MAX_MRQS] = {0};
3414        irqreturn_t result = IRQ_NONE;
3415        struct sdhci_host *host = dev_id;
3416        u32 intmask, mask, unexpected = 0;
3417        int max_loops = 16;
3418        int i;
3419
3420        spin_lock(&host->lock);
3421
3422        if (host->runtime_suspended) {
3423                spin_unlock(&host->lock);
3424                return IRQ_NONE;
3425        }
3426
3427        intmask = sdhci_readl(host, SDHCI_INT_STATUS);
3428        if (!intmask || intmask == 0xffffffff) {
3429                result = IRQ_NONE;
3430                goto out;
3431        }
3432
3433        do {
3434                DBG("IRQ status 0x%08x\n", intmask);
3435
3436                if (host->ops->irq) {
3437                        intmask = host->ops->irq(host, intmask);
3438                        if (!intmask)
3439                                goto cont;
3440                }
3441
3442                /* Clear selected interrupts. */
3443                mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
3444                                  SDHCI_INT_BUS_POWER);
3445                sdhci_writel(host, mask, SDHCI_INT_STATUS);
3446
3447                if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
3448                        u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
3449                                      SDHCI_CARD_PRESENT;
3450
3451                        /*
3452                         * There is a observation on i.mx esdhc.  INSERT
3453                         * bit will be immediately set again when it gets
3454                         * cleared, if a card is inserted.  We have to mask
3455                         * the irq to prevent interrupt storm which will
3456                         * freeze the system.  And the REMOVE gets the
3457                         * same situation.
3458                         *
3459                         * More testing are needed here to ensure it works
3460                         * for other platforms though.
3461                         */
3462                        host->ier &= ~(SDHCI_INT_CARD_INSERT |
3463                                       SDHCI_INT_CARD_REMOVE);
3464                        host->ier |= present ? SDHCI_INT_CARD_REMOVE :
3465                                               SDHCI_INT_CARD_INSERT;
3466                        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3467                        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3468
3469                        sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
3470                                     SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
3471
3472                        host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
3473                                                       SDHCI_INT_CARD_REMOVE);
3474                        result = IRQ_WAKE_THREAD;
3475                }
3476
3477                if (intmask & SDHCI_INT_CMD_MASK)
3478                        sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask);
3479
3480                if (intmask & SDHCI_INT_DATA_MASK)
3481                        sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
3482
3483                if (intmask & SDHCI_INT_BUS_POWER)
3484                        pr_err("%s: Card is consuming too much power!\n",
3485                                mmc_hostname(host->mmc));
3486
3487                if (intmask & SDHCI_INT_RETUNE)
3488                        mmc_retune_needed(host->mmc);
3489
3490                if ((intmask & SDHCI_INT_CARD_INT) &&
3491                    (host->ier & SDHCI_INT_CARD_INT)) {
3492                        sdhci_enable_sdio_irq_nolock(host, false);
3493                        sdio_signal_irq(host->mmc);
3494                }
3495
3496                intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
3497                             SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
3498                             SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
3499                             SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
3500
3501                if (intmask) {
3502                        unexpected |= intmask;
3503                        sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3504                }
3505cont:
3506                if (result == IRQ_NONE)
3507                        result = IRQ_HANDLED;
3508
3509                intmask = sdhci_readl(host, SDHCI_INT_STATUS);
3510        } while (intmask && --max_loops);
3511
3512        /* Determine if mrqs can be completed immediately */
3513        for (i = 0; i < SDHCI_MAX_MRQS; i++) {
3514                struct mmc_request *mrq = host->mrqs_done[i];
3515
3516                if (!mrq)
3517                        continue;
3518
3519                if (sdhci_defer_done(host, mrq)) {
3520                        result = IRQ_WAKE_THREAD;
3521                } else {
3522                        mrqs_done[i] = mrq;
3523                        host->mrqs_done[i] = NULL;
3524                }
3525        }
3526out:
3527        if (host->deferred_cmd)
3528                result = IRQ_WAKE_THREAD;
3529
3530        spin_unlock(&host->lock);
3531
3532        /* Process mrqs ready for immediate completion */
3533        for (i = 0; i < SDHCI_MAX_MRQS; i++) {
3534                if (!mrqs_done[i])
3535                        continue;
3536
3537                if (host->ops->request_done)
3538                        host->ops->request_done(host, mrqs_done[i]);
3539                else
3540                        mmc_request_done(host->mmc, mrqs_done[i]);
3541        }
3542
3543        if (unexpected) {
3544                pr_err("%s: Unexpected interrupt 0x%08x.\n",
3545                           mmc_hostname(host->mmc), unexpected);
3546                sdhci_dumpregs(host);
3547        }
3548
3549        return result;
3550}
3551
3552static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
3553{
3554        struct sdhci_host *host = dev_id;
3555        struct mmc_command *cmd;
3556        unsigned long flags;
3557        u32 isr;
3558
3559        while (!sdhci_request_done(host))
3560                ;
3561
3562        spin_lock_irqsave(&host->lock, flags);
3563
3564        isr = host->thread_isr;
3565        host->thread_isr = 0;
3566
3567        cmd = host->deferred_cmd;
3568        if (cmd && !sdhci_send_command_retry(host, cmd, flags))
3569                sdhci_finish_mrq(host, cmd->mrq);
3570
3571        spin_unlock_irqrestore(&host->lock, flags);
3572
3573        if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
3574                struct mmc_host *mmc = host->mmc;
3575
3576                mmc->ops->card_event(mmc);
3577                mmc_detect_change(mmc, msecs_to_jiffies(200));
3578        }
3579
3580        return IRQ_HANDLED;
3581}
3582
3583/*****************************************************************************\
3584 *                                                                           *
3585 * Suspend/resume                                                            *
3586 *                                                                           *
3587\*****************************************************************************/
3588
3589#ifdef CONFIG_PM
3590
3591static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host)
3592{
3593        return mmc_card_is_removable(host->mmc) &&
3594               !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3595               !mmc_can_gpio_cd(host->mmc);
3596}
3597
3598/*
3599 * To enable wakeup events, the corresponding events have to be enabled in
3600 * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
3601 * Table' in the SD Host Controller Standard Specification.
3602 * It is useless to restore SDHCI_INT_ENABLE state in
3603 * sdhci_disable_irq_wakeups() since it will be set by
3604 * sdhci_enable_card_detection() or sdhci_init().
3605 */
3606static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
3607{
3608        u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
3609                  SDHCI_WAKE_ON_INT;
3610        u32 irq_val = 0;
3611        u8 wake_val = 0;
3612        u8 val;
3613
3614        if (sdhci_cd_irq_can_wakeup(host)) {
3615                wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
3616                irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
3617        }
3618
3619        if (mmc_card_wake_sdio_irq(host->mmc)) {
3620                wake_val |= SDHCI_WAKE_ON_INT;
3621                irq_val |= SDHCI_INT_CARD_INT;
3622        }
3623
3624        if (!irq_val)
3625                return false;
3626
3627        val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3628        val &= ~mask;
3629        val |= wake_val;
3630        sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3631
3632        sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
3633
3634        host->irq_wake_enabled = !enable_irq_wake(host->irq);
3635
3636        return host->irq_wake_enabled;
3637}
3638
3639static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
3640{
3641        u8 val;
3642        u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
3643                        | SDHCI_WAKE_ON_INT;
3644
3645        val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3646        val &= ~mask;
3647        sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3648
3649        disable_irq_wake(host->irq);
3650
3651        host->irq_wake_enabled = false;
3652}
3653
3654int sdhci_suspend_host(struct sdhci_host *host)
3655{
3656        sdhci_disable_card_detection(host);
3657
3658        mmc_retune_timer_stop(host->mmc);
3659
3660        if (!device_may_wakeup(mmc_dev(host->mmc)) ||
3661            !sdhci_enable_irq_wakeups(host)) {
3662                host->ier = 0;
3663                sdhci_writel(host, 0, SDHCI_INT_ENABLE);
3664                sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
3665                free_irq(host->irq, host);
3666        }
3667
3668        return 0;
3669}
3670
3671EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3672
3673int sdhci_resume_host(struct sdhci_host *host)
3674{
3675        struct mmc_host *mmc = host->mmc;
3676        int ret = 0;
3677
3678        if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3679                if (host->ops->enable_dma)
3680                        host->ops->enable_dma(host);
3681        }
3682
3683        if ((mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3684            (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3685                /* Card keeps power but host controller does not */
3686                sdhci_init(host, 0);
3687                host->pwr = 0;
3688                host->clock = 0;
3689                mmc->ops->set_ios(mmc, &mmc->ios);
3690        } else {
3691                sdhci_init(host, (mmc->pm_flags & MMC_PM_KEEP_POWER));
3692        }
3693
3694        if (host->irq_wake_enabled) {
3695                sdhci_disable_irq_wakeups(host);
3696        } else {
3697                ret = request_threaded_irq(host->irq, sdhci_irq,
3698                                           sdhci_thread_irq, IRQF_SHARED,
3699                                           mmc_hostname(mmc), host);
3700                if (ret)
3701                        return ret;
3702        }
3703
3704        sdhci_enable_card_detection(host);
3705
3706        return ret;
3707}
3708
3709EXPORT_SYMBOL_GPL(sdhci_resume_host);
3710
3711int sdhci_runtime_suspend_host(struct sdhci_host *host)
3712{
3713        unsigned long flags;
3714
3715        mmc_retune_timer_stop(host->mmc);
3716
3717        spin_lock_irqsave(&host->lock, flags);
3718        host->ier &= SDHCI_INT_CARD_INT;
3719        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3720        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3721        spin_unlock_irqrestore(&host->lock, flags);
3722
3723        synchronize_hardirq(host->irq);
3724
3725        spin_lock_irqsave(&host->lock, flags);
3726        host->runtime_suspended = true;
3727        spin_unlock_irqrestore(&host->lock, flags);
3728
3729        return 0;
3730}
3731EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3732
3733int sdhci_runtime_resume_host(struct sdhci_host *host, int soft_reset)
3734{
3735        struct mmc_host *mmc = host->mmc;
3736        unsigned long flags;
3737        int host_flags = host->flags;
3738
3739        if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3740                if (host->ops->enable_dma)
3741                        host->ops->enable_dma(host);
3742        }
3743
3744        sdhci_init(host, soft_reset);
3745
3746        if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3747            mmc->ios.power_mode != MMC_POWER_OFF) {
3748                /* Force clock and power re-program */
3749                host->pwr = 0;
3750                host->clock = 0;
3751                mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3752                mmc->ops->set_ios(mmc, &mmc->ios);
3753
3754                if ((host_flags & SDHCI_PV_ENABLED) &&
3755                    !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3756                        spin_lock_irqsave(&host->lock, flags);
3757                        sdhci_enable_preset_value(host, true);
3758                        spin_unlock_irqrestore(&host->lock, flags);
3759                }
3760
3761                if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3762                    mmc->ops->hs400_enhanced_strobe)
3763                        mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3764        }
3765
3766        spin_lock_irqsave(&host->lock, flags);
3767
3768        host->runtime_suspended = false;
3769
3770        /* Enable SDIO IRQ */
3771        if (sdio_irq_claimed(mmc))
3772                sdhci_enable_sdio_irq_nolock(host, true);
3773
3774        /* Enable Card Detection */
3775        sdhci_enable_card_detection(host);
3776
3777        spin_unlock_irqrestore(&host->lock, flags);
3778
3779        return 0;
3780}
3781EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3782
3783#endif /* CONFIG_PM */
3784
3785/*****************************************************************************\
3786 *                                                                           *
3787 * Command Queue Engine (CQE) helpers                                        *
3788 *                                                                           *
3789\*****************************************************************************/
3790
3791void sdhci_cqe_enable(struct mmc_host *mmc)
3792{
3793        struct sdhci_host *host = mmc_priv(mmc);
3794        unsigned long flags;
3795        u8 ctrl;
3796
3797        spin_lock_irqsave(&host->lock, flags);
3798
3799        ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3800        ctrl &= ~SDHCI_CTRL_DMA_MASK;
3801        /*
3802         * Host from V4.10 supports ADMA3 DMA type.
3803         * ADMA3 performs integrated descriptor which is more suitable
3804         * for cmd queuing to fetch both command and transfer descriptors.
3805         */
3806        if (host->v4_mode && (host->caps1 & SDHCI_CAN_DO_ADMA3))
3807                ctrl |= SDHCI_CTRL_ADMA3;
3808        else if (host->flags & SDHCI_USE_64_BIT_DMA)
3809                ctrl |= SDHCI_CTRL_ADMA64;
3810        else
3811                ctrl |= SDHCI_CTRL_ADMA32;
3812        sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3813
3814        sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3815                     SDHCI_BLOCK_SIZE);
3816
3817        /* Set maximum timeout */
3818        sdhci_set_timeout(host, NULL);
3819
3820        host->ier = host->cqe_ier;
3821
3822        sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3823        sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3824
3825        host->cqe_on = true;
3826
3827        pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3828                 mmc_hostname(mmc), host->ier,
3829                 sdhci_readl(host, SDHCI_INT_STATUS));
3830
3831        spin_unlock_irqrestore(&host->lock, flags);
3832}
3833EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3834
3835void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3836{
3837        struct sdhci_host *host = mmc_priv(mmc);
3838        unsigned long flags;
3839
3840        spin_lock_irqsave(&host->lock, flags);
3841
3842        sdhci_set_default_irqs(host);
3843
3844        host->cqe_on = false;
3845
3846        if (recovery) {
3847                sdhci_do_reset(host, SDHCI_RESET_CMD);
3848                sdhci_do_reset(host, SDHCI_RESET_DATA);
3849        }
3850
3851        pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3852                 mmc_hostname(mmc), host->ier,
3853                 sdhci_readl(host, SDHCI_INT_STATUS));
3854
3855        spin_unlock_irqrestore(&host->lock, flags);
3856}
3857EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3858
3859bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3860                   int *data_error)
3861{
3862        u32 mask;
3863
3864        if (!host->cqe_on)
3865                return false;
3866
3867        if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3868                *cmd_error = -EILSEQ;
3869        else if (intmask & SDHCI_INT_TIMEOUT)
3870                *cmd_error = -ETIMEDOUT;
3871        else
3872                *cmd_error = 0;
3873
3874        if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3875                *data_error = -EILSEQ;
3876        else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3877                *data_error = -ETIMEDOUT;
3878        else if (intmask & SDHCI_INT_ADMA_ERROR)
3879                *data_error = -EIO;
3880        else
3881                *data_error = 0;
3882
3883        /* Clear selected interrupts. */
3884        mask = intmask & host->cqe_ier;
3885        sdhci_writel(host, mask, SDHCI_INT_STATUS);
3886
3887        if (intmask & SDHCI_INT_BUS_POWER)
3888                pr_err("%s: Card is consuming too much power!\n",
3889                       mmc_hostname(host->mmc));
3890
3891        intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3892        if (intmask) {
3893                sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3894                pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3895                       mmc_hostname(host->mmc), intmask);
3896                sdhci_dumpregs(host);
3897        }
3898
3899        return true;
3900}
3901EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3902
3903/*****************************************************************************\
3904 *                                                                           *
3905 * Device allocation/registration                                            *
3906 *                                                                           *
3907\*****************************************************************************/
3908
3909struct sdhci_host *sdhci_alloc_host(struct device *dev,
3910        size_t priv_size)
3911{
3912        struct mmc_host *mmc;
3913        struct sdhci_host *host;
3914
3915        WARN_ON(dev == NULL);
3916
3917        mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3918        if (!mmc)
3919                return ERR_PTR(-ENOMEM);
3920
3921        host = mmc_priv(mmc);
3922        host->mmc = mmc;
3923        host->mmc_host_ops = sdhci_ops;
3924        mmc->ops = &host->mmc_host_ops;
3925
3926        host->flags = SDHCI_SIGNALING_330;
3927
3928        host->cqe_ier     = SDHCI_CQE_INT_MASK;
3929        host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3930
3931        host->tuning_delay = -1;
3932        host->tuning_loop_count = MAX_TUNING_LOOP;
3933
3934        host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3935
3936        /*
3937         * The DMA table descriptor count is calculated as the maximum
3938         * number of segments times 2, to allow for an alignment
3939         * descriptor for each segment, plus 1 for a nop end descriptor.
3940         */
3941        host->adma_table_cnt = SDHCI_MAX_SEGS * 2 + 1;
3942
3943        return host;
3944}
3945
3946EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3947
3948static int sdhci_set_dma_mask(struct sdhci_host *host)
3949{
3950        struct mmc_host *mmc = host->mmc;
3951        struct device *dev = mmc_dev(mmc);
3952        int ret = -EINVAL;
3953
3954        if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3955                host->flags &= ~SDHCI_USE_64_BIT_DMA;
3956
3957        /* Try 64-bit mask if hardware is capable  of it */
3958        if (host->flags & SDHCI_USE_64_BIT_DMA) {
3959                ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3960                if (ret) {
3961                        pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3962                                mmc_hostname(mmc));
3963                        host->flags &= ~SDHCI_USE_64_BIT_DMA;
3964                }
3965        }
3966
3967        /* 32-bit mask as default & fallback */
3968        if (ret) {
3969                ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3970                if (ret)
3971                        pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3972                                mmc_hostname(mmc));
3973        }
3974
3975        return ret;
3976}
3977
3978void __sdhci_read_caps(struct sdhci_host *host, const u16 *ver,
3979                       const u32 *caps, const u32 *caps1)
3980{
3981        u16 v;
3982        u64 dt_caps_mask = 0;
3983        u64 dt_caps = 0;
3984
3985        if (host->read_caps)
3986                return;
3987
3988        host->read_caps = true;
3989
3990        if (debug_quirks)
3991                host->quirks = debug_quirks;
3992
3993        if (debug_quirks2)
3994                host->quirks2 = debug_quirks2;
3995
3996        sdhci_do_reset(host, SDHCI_RESET_ALL);
3997
3998        if (host->v4_mode)
3999                sdhci_do_enable_v4_mode(host);
4000
4001        device_property_read_u64(mmc_dev(host->mmc),
4002                                 "sdhci-caps-mask", &dt_caps_mask);
4003        device_property_read_u64(mmc_dev(host->mmc),
4004                                 "sdhci-caps", &dt_caps);
4005
4006        v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
4007        host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
4008
4009        if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
4010                return;
4011
4012        if (caps) {
4013                host->caps = *caps;
4014        } else {
4015                host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
4016                host->caps &= ~lower_32_bits(dt_caps_mask);
4017                host->caps |= lower_32_bits(dt_caps);
4018        }
4019
4020        if (host->version < SDHCI_SPEC_300)
4021                return;
4022
4023        if (caps1) {
4024                host->caps1 = *caps1;
4025        } else {
4026                host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
4027                host->caps1 &= ~upper_32_bits(dt_caps_mask);
4028                host->caps1 |= upper_32_bits(dt_caps);
4029        }
4030}
4031EXPORT_SYMBOL_GPL(__sdhci_read_caps);
4032
4033static void sdhci_allocate_bounce_buffer(struct sdhci_host *host)
4034{
4035        struct mmc_host *mmc = host->mmc;
4036        unsigned int max_blocks;
4037        unsigned int bounce_size;
4038        int ret;
4039
4040        /*
4041         * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
4042         * has diminishing returns, this is probably because SD/MMC
4043         * cards are usually optimized to handle this size of requests.
4044         */
4045        bounce_size = SZ_64K;
4046        /*
4047         * Adjust downwards to maximum request size if this is less
4048         * than our segment size, else hammer down the maximum
4049         * request size to the maximum buffer size.
4050         */
4051        if (mmc->max_req_size < bounce_size)
4052                bounce_size = mmc->max_req_size;
4053        max_blocks = bounce_size / 512;
4054
4055        /*
4056         * When we just support one segment, we can get significant
4057         * speedups by the help of a bounce buffer to group scattered
4058         * reads/writes together.
4059         */
4060        host->bounce_buffer = devm_kmalloc(mmc_dev(mmc),
4061                                           bounce_size,
4062                                           GFP_KERNEL);
4063        if (!host->bounce_buffer) {
4064                pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
4065                       mmc_hostname(mmc),
4066                       bounce_size);
4067                /*
4068                 * Exiting with zero here makes sure we proceed with
4069                 * mmc->max_segs == 1.
4070                 */
4071                return;
4072        }
4073
4074        host->bounce_addr = dma_map_single(mmc_dev(mmc),
4075                                           host->bounce_buffer,
4076                                           bounce_size,
4077                                           DMA_BIDIRECTIONAL);
4078        ret = dma_mapping_error(mmc_dev(mmc), host->bounce_addr);
4079        if (ret) {
4080                devm_kfree(mmc_dev(mmc), host->bounce_buffer);
4081                host->bounce_buffer = NULL;
4082                /* Again fall back to max_segs == 1 */
4083                return;
4084        }
4085
4086        host->bounce_buffer_size = bounce_size;
4087
4088        /* Lie about this since we're bouncing */
4089        mmc->max_segs = max_blocks;
4090        mmc->max_seg_size = bounce_size;
4091        mmc->max_req_size = bounce_size;
4092
4093        pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
4094                mmc_hostname(mmc), max_blocks, bounce_size);
4095}
4096
4097static inline bool sdhci_can_64bit_dma(struct sdhci_host *host)
4098{
4099        /*
4100         * According to SD Host Controller spec v4.10, bit[27] added from
4101         * version 4.10 in Capabilities Register is used as 64-bit System
4102         * Address support for V4 mode.
4103         */
4104        if (host->version >= SDHCI_SPEC_410 && host->v4_mode)
4105                return host->caps & SDHCI_CAN_64BIT_V4;
4106
4107        return host->caps & SDHCI_CAN_64BIT;
4108}
4109
4110int sdhci_setup_host(struct sdhci_host *host)
4111{
4112        struct mmc_host *mmc;
4113        u32 max_current_caps;
4114        unsigned int ocr_avail;
4115        unsigned int override_timeout_clk;
4116        u32 max_clk;
4117        int ret = 0;
4118        bool enable_vqmmc = false;
4119
4120        WARN_ON(host == NULL);
4121        if (host == NULL)
4122                return -EINVAL;
4123
4124        mmc = host->mmc;
4125
4126        /*
4127         * If there are external regulators, get them. Note this must be done
4128         * early before resetting the host and reading the capabilities so that
4129         * the host can take the appropriate action if regulators are not
4130         * available.
4131         */
4132        if (!mmc->supply.vqmmc) {
4133                ret = mmc_regulator_get_supply(mmc);
4134                if (ret)
4135                        return ret;
4136                enable_vqmmc  = true;
4137        }
4138
4139        DBG("Version:   0x%08x | Present:  0x%08x\n",
4140            sdhci_readw(host, SDHCI_HOST_VERSION),
4141            sdhci_readl(host, SDHCI_PRESENT_STATE));
4142        DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
4143            sdhci_readl(host, SDHCI_CAPABILITIES),
4144            sdhci_readl(host, SDHCI_CAPABILITIES_1));
4145
4146        sdhci_read_caps(host);
4147
4148        override_timeout_clk = host->timeout_clk;
4149
4150        if (host->version > SDHCI_SPEC_420) {
4151                pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
4152                       mmc_hostname(mmc), host->version);
4153        }
4154
4155        if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
4156                host->flags |= SDHCI_USE_SDMA;
4157        else if (!(host->caps & SDHCI_CAN_DO_SDMA))
4158                DBG("Controller doesn't have SDMA capability\n");
4159        else
4160                host->flags |= SDHCI_USE_SDMA;
4161
4162        if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
4163                (host->flags & SDHCI_USE_SDMA)) {
4164                DBG("Disabling DMA as it is marked broken\n");
4165                host->flags &= ~SDHCI_USE_SDMA;
4166        }
4167
4168        if ((host->version >= SDHCI_SPEC_200) &&
4169                (host->caps & SDHCI_CAN_DO_ADMA2))
4170                host->flags |= SDHCI_USE_ADMA;
4171
4172        if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
4173                (host->flags & SDHCI_USE_ADMA)) {
4174                DBG("Disabling ADMA as it is marked broken\n");
4175                host->flags &= ~SDHCI_USE_ADMA;
4176        }
4177
4178        if (sdhci_can_64bit_dma(host))
4179                host->flags |= SDHCI_USE_64_BIT_DMA;
4180
4181        if (host->use_external_dma) {
4182                ret = sdhci_external_dma_init(host);
4183                if (ret == -EPROBE_DEFER)
4184                        goto unreg;
4185                /*
4186                 * Fall back to use the DMA/PIO integrated in standard SDHCI
4187                 * instead of external DMA devices.
4188                 */
4189                else if (ret)
4190                        sdhci_switch_external_dma(host, false);
4191                /* Disable internal DMA sources */
4192                else
4193                        host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
4194        }
4195
4196        if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
4197                if (host->ops->set_dma_mask)
4198                        ret = host->ops->set_dma_mask(host);
4199                else
4200                        ret = sdhci_set_dma_mask(host);
4201
4202                if (!ret && host->ops->enable_dma)
4203                        ret = host->ops->enable_dma(host);
4204
4205                if (ret) {
4206                        pr_warn("%s: No suitable DMA available - falling back to PIO\n",
4207                                mmc_hostname(mmc));
4208                        host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
4209
4210                        ret = 0;
4211                }
4212        }
4213
4214        /* SDMA does not support 64-bit DMA if v4 mode not set */
4215        if ((host->flags & SDHCI_USE_64_BIT_DMA) && !host->v4_mode)
4216                host->flags &= ~SDHCI_USE_SDMA;
4217
4218        if (host->flags & SDHCI_USE_ADMA) {
4219                dma_addr_t dma;
4220                void *buf;
4221
4222                if (!(host->flags & SDHCI_USE_64_BIT_DMA))
4223                        host->alloc_desc_sz = SDHCI_ADMA2_32_DESC_SZ;
4224                else if (!host->alloc_desc_sz)
4225                        host->alloc_desc_sz = SDHCI_ADMA2_64_DESC_SZ(host);
4226
4227                host->desc_sz = host->alloc_desc_sz;
4228                host->adma_table_sz = host->adma_table_cnt * host->desc_sz;
4229
4230                host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
4231                /*
4232                 * Use zalloc to zero the reserved high 32-bits of 128-bit
4233                 * descriptors so that they never need to be written.
4234                 */
4235                buf = dma_alloc_coherent(mmc_dev(mmc),
4236                                         host->align_buffer_sz + host->adma_table_sz,
4237                                         &dma, GFP_KERNEL);
4238                if (!buf) {
4239                        pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
4240                                mmc_hostname(mmc));
4241                        host->flags &= ~SDHCI_USE_ADMA;
4242                } else if ((dma + host->align_buffer_sz) &
4243                           (SDHCI_ADMA2_DESC_ALIGN - 1)) {
4244                        pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
4245                                mmc_hostname(mmc));
4246                        host->flags &= ~SDHCI_USE_ADMA;
4247                        dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4248                                          host->adma_table_sz, buf, dma);
4249                } else {
4250                        host->align_buffer = buf;
4251                        host->align_addr = dma;
4252
4253                        host->adma_table = buf + host->align_buffer_sz;
4254                        host->adma_addr = dma + host->align_buffer_sz;
4255                }
4256        }
4257
4258        /*
4259         * If we use DMA, then it's up to the caller to set the DMA
4260         * mask, but PIO does not need the hw shim so we set a new
4261         * mask here in that case.
4262         */
4263        if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
4264                host->dma_mask = DMA_BIT_MASK(64);
4265                mmc_dev(mmc)->dma_mask = &host->dma_mask;
4266        }
4267
4268        if (host->version >= SDHCI_SPEC_300)
4269                host->max_clk = FIELD_GET(SDHCI_CLOCK_V3_BASE_MASK, host->caps);
4270        else
4271                host->max_clk = FIELD_GET(SDHCI_CLOCK_BASE_MASK, host->caps);
4272
4273        host->max_clk *= 1000000;
4274        if (host->max_clk == 0 || host->quirks &
4275                        SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
4276                if (!host->ops->get_max_clock) {
4277                        pr_err("%s: Hardware doesn't specify base clock frequency.\n",
4278                               mmc_hostname(mmc));
4279                        ret = -ENODEV;
4280                        goto undma;
4281                }
4282                host->max_clk = host->ops->get_max_clock(host);
4283        }
4284
4285        /*
4286         * In case of Host Controller v3.00, find out whether clock
4287         * multiplier is supported.
4288         */
4289        host->clk_mul = FIELD_GET(SDHCI_CLOCK_MUL_MASK, host->caps1);
4290
4291        /*
4292         * In case the value in Clock Multiplier is 0, then programmable
4293         * clock mode is not supported, otherwise the actual clock
4294         * multiplier is one more than the value of Clock Multiplier
4295         * in the Capabilities Register.
4296         */
4297        if (host->clk_mul)
4298                host->clk_mul += 1;
4299
4300        /*
4301         * Set host parameters.
4302         */
4303        max_clk = host->max_clk;
4304
4305        if (host->ops->get_min_clock)
4306                mmc->f_min = host->ops->get_min_clock(host);
4307        else if (host->version >= SDHCI_SPEC_300) {
4308                if (host->clk_mul)
4309                        max_clk = host->max_clk * host->clk_mul;
4310                /*
4311                 * Divided Clock Mode minimum clock rate is always less than
4312                 * Programmable Clock Mode minimum clock rate.
4313                 */
4314                mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
4315        } else
4316                mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
4317
4318        if (!mmc->f_max || mmc->f_max > max_clk)
4319                mmc->f_max = max_clk;
4320
4321        if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
4322                host->timeout_clk = FIELD_GET(SDHCI_TIMEOUT_CLK_MASK, host->caps);
4323
4324                if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
4325                        host->timeout_clk *= 1000;
4326
4327                if (host->timeout_clk == 0) {
4328                        if (!host->ops->get_timeout_clock) {
4329                                pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
4330                                        mmc_hostname(mmc));
4331                                ret = -ENODEV;
4332                                goto undma;
4333                        }
4334
4335                        host->timeout_clk =
4336                                DIV_ROUND_UP(host->ops->get_timeout_clock(host),
4337                                             1000);
4338                }
4339
4340                if (override_timeout_clk)
4341                        host->timeout_clk = override_timeout_clk;
4342
4343                mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
4344                        host->ops->get_max_timeout_count(host) : 1 << 27;
4345                mmc->max_busy_timeout /= host->timeout_clk;
4346        }
4347
4348        if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&
4349            !host->ops->get_max_timeout_count)
4350                mmc->max_busy_timeout = 0;
4351
4352        mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_CMD23;
4353        mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
4354
4355        if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
4356                host->flags |= SDHCI_AUTO_CMD12;
4357
4358        /*
4359         * For v3 mode, Auto-CMD23 stuff only works in ADMA or PIO.
4360         * For v4 mode, SDMA may use Auto-CMD23 as well.
4361         */
4362        if ((host->version >= SDHCI_SPEC_300) &&
4363            ((host->flags & SDHCI_USE_ADMA) ||
4364             !(host->flags & SDHCI_USE_SDMA) || host->v4_mode) &&
4365             !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
4366                host->flags |= SDHCI_AUTO_CMD23;
4367                DBG("Auto-CMD23 available\n");
4368        } else {
4369                DBG("Auto-CMD23 unavailable\n");
4370        }
4371
4372        /*
4373         * A controller may support 8-bit width, but the board itself
4374         * might not have the pins brought out.  Boards that support
4375         * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
4376         * their platform code before calling sdhci_add_host(), and we
4377         * won't assume 8-bit width for hosts without that CAP.
4378         */
4379        if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
4380                mmc->caps |= MMC_CAP_4_BIT_DATA;
4381
4382        if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
4383                mmc->caps &= ~MMC_CAP_CMD23;
4384
4385        if (host->caps & SDHCI_CAN_DO_HISPD)
4386                mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
4387
4388        if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
4389            mmc_card_is_removable(mmc) &&
4390            mmc_gpio_get_cd(mmc) < 0)
4391                mmc->caps |= MMC_CAP_NEEDS_POLL;
4392
4393        if (!IS_ERR(mmc->supply.vqmmc)) {
4394                if (enable_vqmmc) {
4395                        ret = regulator_enable(mmc->supply.vqmmc);
4396                        host->sdhci_core_to_disable_vqmmc = !ret;
4397                }
4398
4399                /* If vqmmc provides no 1.8V signalling, then there's no UHS */
4400                if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
4401                                                    1950000))
4402                        host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
4403                                         SDHCI_SUPPORT_SDR50 |
4404                                         SDHCI_SUPPORT_DDR50);
4405
4406                /* In eMMC case vqmmc might be a fixed 1.8V regulator */
4407                if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000,
4408                                                    3600000))
4409                        host->flags &= ~SDHCI_SIGNALING_330;
4410
4411                if (ret) {
4412                        pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
4413                                mmc_hostname(mmc), ret);
4414                        mmc->supply.vqmmc = ERR_PTR(-EINVAL);
4415                }
4416
4417        }
4418
4419        if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
4420                host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
4421                                 SDHCI_SUPPORT_DDR50);
4422                /*
4423                 * The SDHCI controller in a SoC might support HS200/HS400
4424                 * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),
4425                 * but if the board is modeled such that the IO lines are not
4426                 * connected to 1.8v then HS200/HS400 cannot be supported.
4427                 * Disable HS200/HS400 if the board does not have 1.8v connected
4428                 * to the IO lines. (Applicable for other modes in 1.8v)
4429                 */
4430                mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);
4431                mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);
4432        }
4433
4434        /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
4435        if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
4436                           <