linux/drivers/usb/dwc2/gadget.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2011 Samsung Electronics Co., Ltd.
   4 *              http://www.samsung.com
   5 *
   6 * Copyright 2008 Openmoko, Inc.
   7 * Copyright 2008 Simtec Electronics
   8 *      Ben Dooks <ben@simtec.co.uk>
   9 *      http://armlinux.simtec.co.uk/
  10 *
  11 * S3C USB2.0 High-speed / OtG driver
  12 */
  13
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/spinlock.h>
  17#include <linux/interrupt.h>
  18#include <linux/platform_device.h>
  19#include <linux/dma-mapping.h>
  20#include <linux/mutex.h>
  21#include <linux/seq_file.h>
  22#include <linux/delay.h>
  23#include <linux/io.h>
  24#include <linux/slab.h>
  25#include <linux/of_platform.h>
  26
  27#include <linux/usb/ch9.h>
  28#include <linux/usb/gadget.h>
  29#include <linux/usb/phy.h>
  30#include <linux/usb/composite.h>
  31
  32
  33#include "core.h"
  34#include "hw.h"
  35
  36/* conversion functions */
  37static inline struct dwc2_hsotg_req *our_req(struct usb_request *req)
  38{
  39        return container_of(req, struct dwc2_hsotg_req, req);
  40}
  41
  42static inline struct dwc2_hsotg_ep *our_ep(struct usb_ep *ep)
  43{
  44        return container_of(ep, struct dwc2_hsotg_ep, ep);
  45}
  46
  47static inline struct dwc2_hsotg *to_hsotg(struct usb_gadget *gadget)
  48{
  49        return container_of(gadget, struct dwc2_hsotg, gadget);
  50}
  51
  52static inline void dwc2_set_bit(struct dwc2_hsotg *hsotg, u32 offset, u32 val)
  53{
  54        dwc2_writel(hsotg, dwc2_readl(hsotg, offset) | val, offset);
  55}
  56
  57static inline void dwc2_clear_bit(struct dwc2_hsotg *hsotg, u32 offset, u32 val)
  58{
  59        dwc2_writel(hsotg, dwc2_readl(hsotg, offset) & ~val, offset);
  60}
  61
  62static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg,
  63                                                u32 ep_index, u32 dir_in)
  64{
  65        if (dir_in)
  66                return hsotg->eps_in[ep_index];
  67        else
  68                return hsotg->eps_out[ep_index];
  69}
  70
  71/* forward declaration of functions */
  72static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg);
  73
  74/**
  75 * using_dma - return the DMA status of the driver.
  76 * @hsotg: The driver state.
  77 *
  78 * Return true if we're using DMA.
  79 *
  80 * Currently, we have the DMA support code worked into everywhere
  81 * that needs it, but the AMBA DMA implementation in the hardware can
  82 * only DMA from 32bit aligned addresses. This means that gadgets such
  83 * as the CDC Ethernet cannot work as they often pass packets which are
  84 * not 32bit aligned.
  85 *
  86 * Unfortunately the choice to use DMA or not is global to the controller
  87 * and seems to be only settable when the controller is being put through
  88 * a core reset. This means we either need to fix the gadgets to take
  89 * account of DMA alignment, or add bounce buffers (yuerk).
  90 *
  91 * g_using_dma is set depending on dts flag.
  92 */
  93static inline bool using_dma(struct dwc2_hsotg *hsotg)
  94{
  95        return hsotg->params.g_dma;
  96}
  97
  98/*
  99 * using_desc_dma - return the descriptor DMA status of the driver.
 100 * @hsotg: The driver state.
 101 *
 102 * Return true if we're using descriptor DMA.
 103 */
 104static inline bool using_desc_dma(struct dwc2_hsotg *hsotg)
 105{
 106        return hsotg->params.g_dma_desc;
 107}
 108
 109/**
 110 * dwc2_gadget_incr_frame_num - Increments the targeted frame number.
 111 * @hs_ep: The endpoint
 112 *
 113 * This function will also check if the frame number overruns DSTS_SOFFN_LIMIT.
 114 * If an overrun occurs it will wrap the value and set the frame_overrun flag.
 115 */
 116static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep)
 117{
 118        hs_ep->target_frame += hs_ep->interval;
 119        if (hs_ep->target_frame > DSTS_SOFFN_LIMIT) {
 120                hs_ep->frame_overrun = true;
 121                hs_ep->target_frame &= DSTS_SOFFN_LIMIT;
 122        } else {
 123                hs_ep->frame_overrun = false;
 124        }
 125}
 126
 127/**
 128 * dwc2_gadget_dec_frame_num_by_one - Decrements the targeted frame number
 129 *                                    by one.
 130 * @hs_ep: The endpoint.
 131 *
 132 * This function used in service interval based scheduling flow to calculate
 133 * descriptor frame number filed value. For service interval mode frame
 134 * number in descriptor should point to last (u)frame in the interval.
 135 *
 136 */
 137static inline void dwc2_gadget_dec_frame_num_by_one(struct dwc2_hsotg_ep *hs_ep)
 138{
 139        if (hs_ep->target_frame)
 140                hs_ep->target_frame -= 1;
 141        else
 142                hs_ep->target_frame = DSTS_SOFFN_LIMIT;
 143}
 144
 145/**
 146 * dwc2_hsotg_en_gsint - enable one or more of the general interrupt
 147 * @hsotg: The device state
 148 * @ints: A bitmask of the interrupts to enable
 149 */
 150static void dwc2_hsotg_en_gsint(struct dwc2_hsotg *hsotg, u32 ints)
 151{
 152        u32 gsintmsk = dwc2_readl(hsotg, GINTMSK);
 153        u32 new_gsintmsk;
 154
 155        new_gsintmsk = gsintmsk | ints;
 156
 157        if (new_gsintmsk != gsintmsk) {
 158                dev_dbg(hsotg->dev, "gsintmsk now 0x%08x\n", new_gsintmsk);
 159                dwc2_writel(hsotg, new_gsintmsk, GINTMSK);
 160        }
 161}
 162
 163/**
 164 * dwc2_hsotg_disable_gsint - disable one or more of the general interrupt
 165 * @hsotg: The device state
 166 * @ints: A bitmask of the interrupts to enable
 167 */
 168static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints)
 169{
 170        u32 gsintmsk = dwc2_readl(hsotg, GINTMSK);
 171        u32 new_gsintmsk;
 172
 173        new_gsintmsk = gsintmsk & ~ints;
 174
 175        if (new_gsintmsk != gsintmsk)
 176                dwc2_writel(hsotg, new_gsintmsk, GINTMSK);
 177}
 178
 179/**
 180 * dwc2_hsotg_ctrl_epint - enable/disable an endpoint irq
 181 * @hsotg: The device state
 182 * @ep: The endpoint index
 183 * @dir_in: True if direction is in.
 184 * @en: The enable value, true to enable
 185 *
 186 * Set or clear the mask for an individual endpoint's interrupt
 187 * request.
 188 */
 189static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg,
 190                                  unsigned int ep, unsigned int dir_in,
 191                                 unsigned int en)
 192{
 193        unsigned long flags;
 194        u32 bit = 1 << ep;
 195        u32 daint;
 196
 197        if (!dir_in)
 198                bit <<= 16;
 199
 200        local_irq_save(flags);
 201        daint = dwc2_readl(hsotg, DAINTMSK);
 202        if (en)
 203                daint |= bit;
 204        else
 205                daint &= ~bit;
 206        dwc2_writel(hsotg, daint, DAINTMSK);
 207        local_irq_restore(flags);
 208}
 209
 210/**
 211 * dwc2_hsotg_tx_fifo_count - return count of TX FIFOs in device mode
 212 *
 213 * @hsotg: Programming view of the DWC_otg controller
 214 */
 215int dwc2_hsotg_tx_fifo_count(struct dwc2_hsotg *hsotg)
 216{
 217        if (hsotg->hw_params.en_multiple_tx_fifo)
 218                /* In dedicated FIFO mode we need count of IN EPs */
 219                return hsotg->hw_params.num_dev_in_eps;
 220        else
 221                /* In shared FIFO mode we need count of Periodic IN EPs */
 222                return hsotg->hw_params.num_dev_perio_in_ep;
 223}
 224
 225/**
 226 * dwc2_hsotg_tx_fifo_total_depth - return total FIFO depth available for
 227 * device mode TX FIFOs
 228 *
 229 * @hsotg: Programming view of the DWC_otg controller
 230 */
 231int dwc2_hsotg_tx_fifo_total_depth(struct dwc2_hsotg *hsotg)
 232{
 233        int addr;
 234        int tx_addr_max;
 235        u32 np_tx_fifo_size;
 236
 237        np_tx_fifo_size = min_t(u32, hsotg->hw_params.dev_nperio_tx_fifo_size,
 238                                hsotg->params.g_np_tx_fifo_size);
 239
 240        /* Get Endpoint Info Control block size in DWORDs. */
 241        tx_addr_max = hsotg->hw_params.total_fifo_size;
 242
 243        addr = hsotg->params.g_rx_fifo_size + np_tx_fifo_size;
 244        if (tx_addr_max <= addr)
 245                return 0;
 246
 247        return tx_addr_max - addr;
 248}
 249
 250/**
 251 * dwc2_gadget_wkup_alert_handler - Handler for WKUP_ALERT interrupt
 252 *
 253 * @hsotg: Programming view of the DWC_otg controller
 254 *
 255 */
 256static void dwc2_gadget_wkup_alert_handler(struct dwc2_hsotg *hsotg)
 257{
 258        u32 gintsts2;
 259        u32 gintmsk2;
 260
 261        gintsts2 = dwc2_readl(hsotg, GINTSTS2);
 262        gintmsk2 = dwc2_readl(hsotg, GINTMSK2);
 263        gintsts2 &= gintmsk2;
 264
 265        if (gintsts2 & GINTSTS2_WKUP_ALERT_INT) {
 266                dev_dbg(hsotg->dev, "%s: Wkup_Alert_Int\n", __func__);
 267                dwc2_set_bit(hsotg, GINTSTS2, GINTSTS2_WKUP_ALERT_INT);
 268                dwc2_set_bit(hsotg, DCTL, DCTL_RMTWKUPSIG);
 269        }
 270}
 271
 272/**
 273 * dwc2_hsotg_tx_fifo_average_depth - returns average depth of device mode
 274 * TX FIFOs
 275 *
 276 * @hsotg: Programming view of the DWC_otg controller
 277 */
 278int dwc2_hsotg_tx_fifo_average_depth(struct dwc2_hsotg *hsotg)
 279{
 280        int tx_fifo_count;
 281        int tx_fifo_depth;
 282
 283        tx_fifo_depth = dwc2_hsotg_tx_fifo_total_depth(hsotg);
 284
 285        tx_fifo_count = dwc2_hsotg_tx_fifo_count(hsotg);
 286
 287        if (!tx_fifo_count)
 288                return tx_fifo_depth;
 289        else
 290                return tx_fifo_depth / tx_fifo_count;
 291}
 292
 293/**
 294 * dwc2_hsotg_init_fifo - initialise non-periodic FIFOs
 295 * @hsotg: The device instance.
 296 */
 297static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
 298{
 299        unsigned int ep;
 300        unsigned int addr;
 301        int timeout;
 302
 303        u32 val;
 304        u32 *txfsz = hsotg->params.g_tx_fifo_size;
 305
 306        /* Reset fifo map if not correctly cleared during previous session */
 307        WARN_ON(hsotg->fifo_map);
 308        hsotg->fifo_map = 0;
 309
 310        /* set RX/NPTX FIFO sizes */
 311        dwc2_writel(hsotg, hsotg->params.g_rx_fifo_size, GRXFSIZ);
 312        dwc2_writel(hsotg, (hsotg->params.g_rx_fifo_size <<
 313                    FIFOSIZE_STARTADDR_SHIFT) |
 314                    (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT),
 315                    GNPTXFSIZ);
 316
 317        /*
 318         * arange all the rest of the TX FIFOs, as some versions of this
 319         * block have overlapping default addresses. This also ensures
 320         * that if the settings have been changed, then they are set to
 321         * known values.
 322         */
 323
 324        /* start at the end of the GNPTXFSIZ, rounded up */
 325        addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size;
 326
 327        /*
 328         * Configure fifos sizes from provided configuration and assign
 329         * them to endpoints dynamically according to maxpacket size value of
 330         * given endpoint.
 331         */
 332        for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
 333                if (!txfsz[ep])
 334                        continue;
 335                val = addr;
 336                val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT;
 337                WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem,
 338                          "insufficient fifo memory");
 339                addr += txfsz[ep];
 340
 341                dwc2_writel(hsotg, val, DPTXFSIZN(ep));
 342                val = dwc2_readl(hsotg, DPTXFSIZN(ep));
 343        }
 344
 345        dwc2_writel(hsotg, hsotg->hw_params.total_fifo_size |
 346                    addr << GDFIFOCFG_EPINFOBASE_SHIFT,
 347                    GDFIFOCFG);
 348        /*
 349         * according to p428 of the design guide, we need to ensure that
 350         * all fifos are flushed before continuing
 351         */
 352
 353        dwc2_writel(hsotg, GRSTCTL_TXFNUM(0x10) | GRSTCTL_TXFFLSH |
 354               GRSTCTL_RXFFLSH, GRSTCTL);
 355
 356        /* wait until the fifos are both flushed */
 357        timeout = 100;
 358        while (1) {
 359                val = dwc2_readl(hsotg, GRSTCTL);
 360
 361                if ((val & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)) == 0)
 362                        break;
 363
 364                if (--timeout == 0) {
 365                        dev_err(hsotg->dev,
 366                                "%s: timeout flushing fifos (GRSTCTL=%08x)\n",
 367                                __func__, val);
 368                        break;
 369                }
 370
 371                udelay(1);
 372        }
 373
 374        dev_dbg(hsotg->dev, "FIFOs reset, timeout at %d\n", timeout);
 375}
 376
 377/**
 378 * dwc2_hsotg_ep_alloc_request - allocate USB rerequest structure
 379 * @ep: USB endpoint to allocate request for.
 380 * @flags: Allocation flags
 381 *
 382 * Allocate a new USB request structure appropriate for the specified endpoint
 383 */
 384static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep,
 385                                                       gfp_t flags)
 386{
 387        struct dwc2_hsotg_req *req;
 388
 389        req = kzalloc(sizeof(*req), flags);
 390        if (!req)
 391                return NULL;
 392
 393        INIT_LIST_HEAD(&req->queue);
 394
 395        return &req->req;
 396}
 397
 398/**
 399 * is_ep_periodic - return true if the endpoint is in periodic mode.
 400 * @hs_ep: The endpoint to query.
 401 *
 402 * Returns true if the endpoint is in periodic mode, meaning it is being
 403 * used for an Interrupt or ISO transfer.
 404 */
 405static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep)
 406{
 407        return hs_ep->periodic;
 408}
 409
 410/**
 411 * dwc2_hsotg_unmap_dma - unmap the DMA memory being used for the request
 412 * @hsotg: The device state.
 413 * @hs_ep: The endpoint for the request
 414 * @hs_req: The request being processed.
 415 *
 416 * This is the reverse of dwc2_hsotg_map_dma(), called for the completion
 417 * of a request to ensure the buffer is ready for access by the caller.
 418 */
 419static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg,
 420                                 struct dwc2_hsotg_ep *hs_ep,
 421                                struct dwc2_hsotg_req *hs_req)
 422{
 423        struct usb_request *req = &hs_req->req;
 424
 425        usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->map_dir);
 426}
 427
 428/*
 429 * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains
 430 * for Control endpoint
 431 * @hsotg: The device state.
 432 *
 433 * This function will allocate 4 descriptor chains for EP 0: 2 for
 434 * Setup stage, per one for IN and OUT data/status transactions.
 435 */
 436static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg)
 437{
 438        hsotg->setup_desc[0] =
 439                dmam_alloc_coherent(hsotg->dev,
 440                                    sizeof(struct dwc2_dma_desc),
 441                                    &hsotg->setup_desc_dma[0],
 442                                    GFP_KERNEL);
 443        if (!hsotg->setup_desc[0])
 444                goto fail;
 445
 446        hsotg->setup_desc[1] =
 447                dmam_alloc_coherent(hsotg->dev,
 448                                    sizeof(struct dwc2_dma_desc),
 449                                    &hsotg->setup_desc_dma[1],
 450                                    GFP_KERNEL);
 451        if (!hsotg->setup_desc[1])
 452                goto fail;
 453
 454        hsotg->ctrl_in_desc =
 455                dmam_alloc_coherent(hsotg->dev,
 456                                    sizeof(struct dwc2_dma_desc),
 457                                    &hsotg->ctrl_in_desc_dma,
 458                                    GFP_KERNEL);
 459        if (!hsotg->ctrl_in_desc)
 460                goto fail;
 461
 462        hsotg->ctrl_out_desc =
 463                dmam_alloc_coherent(hsotg->dev,
 464                                    sizeof(struct dwc2_dma_desc),
 465                                    &hsotg->ctrl_out_desc_dma,
 466                                    GFP_KERNEL);
 467        if (!hsotg->ctrl_out_desc)
 468                goto fail;
 469
 470        return 0;
 471
 472fail:
 473        return -ENOMEM;
 474}
 475
 476/**
 477 * dwc2_hsotg_write_fifo - write packet Data to the TxFIFO
 478 * @hsotg: The controller state.
 479 * @hs_ep: The endpoint we're going to write for.
 480 * @hs_req: The request to write data for.
 481 *
 482 * This is called when the TxFIFO has some space in it to hold a new
 483 * transmission and we have something to give it. The actual setup of
 484 * the data size is done elsewhere, so all we have to do is to actually
 485 * write the data.
 486 *
 487 * The return value is zero if there is more space (or nothing was done)
 488 * otherwise -ENOSPC is returned if the FIFO space was used up.
 489 *
 490 * This routine is only needed for PIO
 491 */
 492static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
 493                                 struct dwc2_hsotg_ep *hs_ep,
 494                                struct dwc2_hsotg_req *hs_req)
 495{
 496        bool periodic = is_ep_periodic(hs_ep);
 497        u32 gnptxsts = dwc2_readl(hsotg, GNPTXSTS);
 498        int buf_pos = hs_req->req.actual;
 499        int to_write = hs_ep->size_loaded;
 500        void *data;
 501        int can_write;
 502        int pkt_round;
 503        int max_transfer;
 504
 505        to_write -= (buf_pos - hs_ep->last_load);
 506
 507        /* if there's nothing to write, get out early */
 508        if (to_write == 0)
 509                return 0;
 510
 511        if (periodic && !hsotg->dedicated_fifos) {
 512                u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index));
 513                int size_left;
 514                int size_done;
 515
 516                /*
 517                 * work out how much data was loaded so we can calculate
 518                 * how much data is left in the fifo.
 519                 */
 520
 521                size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
 522
 523                /*
 524                 * if shared fifo, we cannot write anything until the
 525                 * previous data has been completely sent.
 526                 */
 527                if (hs_ep->fifo_load != 0) {
 528                        dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
 529                        return -ENOSPC;
 530                }
 531
 532                dev_dbg(hsotg->dev, "%s: left=%d, load=%d, fifo=%d, size %d\n",
 533                        __func__, size_left,
 534                        hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);
 535
 536                /* how much of the data has moved */
 537                size_done = hs_ep->size_loaded - size_left;
 538
 539                /* how much data is left in the fifo */
 540                can_write = hs_ep->fifo_load - size_done;
 541                dev_dbg(hsotg->dev, "%s: => can_write1=%d\n",
 542                        __func__, can_write);
 543
 544                can_write = hs_ep->fifo_size - can_write;
 545                dev_dbg(hsotg->dev, "%s: => can_write2=%d\n",
 546                        __func__, can_write);
 547
 548                if (can_write <= 0) {
 549                        dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
 550                        return -ENOSPC;
 551                }
 552        } else if (hsotg->dedicated_fifos && hs_ep->index != 0) {
 553                can_write = dwc2_readl(hsotg,
 554                                       DTXFSTS(hs_ep->fifo_index));
 555
 556                can_write &= 0xffff;
 557                can_write *= 4;
 558        } else {
 559                if (GNPTXSTS_NP_TXQ_SPC_AVAIL_GET(gnptxsts) == 0) {
 560                        dev_dbg(hsotg->dev,
 561                                "%s: no queue slots available (0x%08x)\n",
 562                                __func__, gnptxsts);
 563
 564                        dwc2_hsotg_en_gsint(hsotg, GINTSTS_NPTXFEMP);
 565                        return -ENOSPC;
 566                }
 567
 568                can_write = GNPTXSTS_NP_TXF_SPC_AVAIL_GET(gnptxsts);
 569                can_write *= 4; /* fifo size is in 32bit quantities. */
 570        }
 571
 572        max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;
 573
 574        dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n",
 575                __func__, gnptxsts, can_write, to_write, max_transfer);
 576
 577        /*
 578         * limit to 512 bytes of data, it seems at least on the non-periodic
 579         * FIFO, requests of >512 cause the endpoint to get stuck with a
 580         * fragment of the end of the transfer in it.
 581         */
 582        if (can_write > 512 && !periodic)
 583                can_write = 512;
 584
 585        /*
 586         * limit the write to one max-packet size worth of data, but allow
 587         * the transfer to return that it did not run out of fifo space
 588         * doing it.
 589         */
 590        if (to_write > max_transfer) {
 591                to_write = max_transfer;
 592
 593                /* it's needed only when we do not use dedicated fifos */
 594                if (!hsotg->dedicated_fifos)
 595                        dwc2_hsotg_en_gsint(hsotg,
 596                                            periodic ? GINTSTS_PTXFEMP :
 597                                           GINTSTS_NPTXFEMP);
 598        }
 599
 600        /* see if we can write data */
 601
 602        if (to_write > can_write) {
 603                to_write = can_write;
 604                pkt_round = to_write % max_transfer;
 605
 606                /*
 607                 * Round the write down to an
 608                 * exact number of packets.
 609                 *
 610                 * Note, we do not currently check to see if we can ever
 611                 * write a full packet or not to the FIFO.
 612                 */
 613
 614                if (pkt_round)
 615                        to_write -= pkt_round;
 616
 617                /*
 618                 * enable correct FIFO interrupt to alert us when there
 619                 * is more room left.
 620                 */
 621
 622                /* it's needed only when we do not use dedicated fifos */
 623                if (!hsotg->dedicated_fifos)
 624                        dwc2_hsotg_en_gsint(hsotg,
 625                                            periodic ? GINTSTS_PTXFEMP :
 626                                           GINTSTS_NPTXFEMP);
 627        }
 628
 629        dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n",
 630                to_write, hs_req->req.length, can_write, buf_pos);
 631
 632        if (to_write <= 0)
 633                return -ENOSPC;
 634
 635        hs_req->req.actual = buf_pos + to_write;
 636        hs_ep->total_data += to_write;
 637
 638        if (periodic)
 639                hs_ep->fifo_load += to_write;
 640
 641        to_write = DIV_ROUND_UP(to_write, 4);
 642        data = hs_req->req.buf + buf_pos;
 643
 644        dwc2_writel_rep(hsotg, EPFIFO(hs_ep->index), data, to_write);
 645
 646        return (to_write >= can_write) ? -ENOSPC : 0;
 647}
 648
 649/**
 650 * get_ep_limit - get the maximum data legnth for this endpoint
 651 * @hs_ep: The endpoint
 652 *
 653 * Return the maximum data that can be queued in one go on a given endpoint
 654 * so that transfers that are too long can be split.
 655 */
 656static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep)
 657{
 658        int index = hs_ep->index;
 659        unsigned int maxsize;
 660        unsigned int maxpkt;
 661
 662        if (index != 0) {
 663                maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1;
 664                maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1;
 665        } else {
 666                maxsize = 64 + 64;
 667                if (hs_ep->dir_in)
 668                        maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1;
 669                else
 670                        maxpkt = 2;
 671        }
 672
 673        /* we made the constant loading easier above by using +1 */
 674        maxpkt--;
 675        maxsize--;
 676
 677        /*
 678         * constrain by packet count if maxpkts*pktsize is greater
 679         * than the length register size.
 680         */
 681
 682        if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)
 683                maxsize = maxpkt * hs_ep->ep.maxpacket;
 684
 685        return maxsize;
 686}
 687
 688/**
 689 * dwc2_hsotg_read_frameno - read current frame number
 690 * @hsotg: The device instance
 691 *
 692 * Return the current frame number
 693 */
 694static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg)
 695{
 696        u32 dsts;
 697
 698        dsts = dwc2_readl(hsotg, DSTS);
 699        dsts &= DSTS_SOFFN_MASK;
 700        dsts >>= DSTS_SOFFN_SHIFT;
 701
 702        return dsts;
 703}
 704
 705/**
 706 * dwc2_gadget_get_chain_limit - get the maximum data payload value of the
 707 * DMA descriptor chain prepared for specific endpoint
 708 * @hs_ep: The endpoint
 709 *
 710 * Return the maximum data that can be queued in one go on a given endpoint
 711 * depending on its descriptor chain capacity so that transfers that
 712 * are too long can be split.
 713 */
 714static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
 715{
 716        const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
 717        int is_isoc = hs_ep->isochronous;
 718        unsigned int maxsize;
 719        u32 mps = hs_ep->ep.maxpacket;
 720        int dir_in = hs_ep->dir_in;
 721
 722        if (is_isoc)
 723                maxsize = (hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
 724                                           DEV_DMA_ISOC_RX_NBYTES_LIMIT) *
 725                                           MAX_DMA_DESC_NUM_HS_ISOC;
 726        else
 727                maxsize = DEV_DMA_NBYTES_LIMIT * MAX_DMA_DESC_NUM_GENERIC;
 728
 729        /* Interrupt OUT EP with mps not multiple of 4 */
 730        if (hs_ep->index)
 731                if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4))
 732                        maxsize = mps * MAX_DMA_DESC_NUM_GENERIC;
 733
 734        return maxsize;
 735}
 736
 737/*
 738 * dwc2_gadget_get_desc_params - get DMA descriptor parameters.
 739 * @hs_ep: The endpoint
 740 * @mask: RX/TX bytes mask to be defined
 741 *
 742 * Returns maximum data payload for one descriptor after analyzing endpoint
 743 * characteristics.
 744 * DMA descriptor transfer bytes limit depends on EP type:
 745 * Control out - MPS,
 746 * Isochronous - descriptor rx/tx bytes bitfield limit,
 747 * Control In/Bulk/Interrupt - multiple of mps. This will allow to not
 748 * have concatenations from various descriptors within one packet.
 749 * Interrupt OUT - if mps not multiple of 4 then a single packet corresponds
 750 * to a single descriptor.
 751 *
 752 * Selects corresponding mask for RX/TX bytes as well.
 753 */
 754static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask)
 755{
 756        const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
 757        u32 mps = hs_ep->ep.maxpacket;
 758        int dir_in = hs_ep->dir_in;
 759        u32 desc_size = 0;
 760
 761        if (!hs_ep->index && !dir_in) {
 762                desc_size = mps;
 763                *mask = DEV_DMA_NBYTES_MASK;
 764        } else if (hs_ep->isochronous) {
 765                if (dir_in) {
 766                        desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT;
 767                        *mask = DEV_DMA_ISOC_TX_NBYTES_MASK;
 768                } else {
 769                        desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT;
 770                        *mask = DEV_DMA_ISOC_RX_NBYTES_MASK;
 771                }
 772        } else {
 773                desc_size = DEV_DMA_NBYTES_LIMIT;
 774                *mask = DEV_DMA_NBYTES_MASK;
 775
 776                /* Round down desc_size to be mps multiple */
 777                desc_size -= desc_size % mps;
 778        }
 779
 780        /* Interrupt OUT EP with mps not multiple of 4 */
 781        if (hs_ep->index)
 782                if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4)) {
 783                        desc_size = mps;
 784                        *mask = DEV_DMA_NBYTES_MASK;
 785                }
 786
 787        return desc_size;
 788}
 789
 790static void dwc2_gadget_fill_nonisoc_xfer_ddma_one(struct dwc2_hsotg_ep *hs_ep,
 791                                                 struct dwc2_dma_desc **desc,
 792                                                 dma_addr_t dma_buff,
 793                                                 unsigned int len,
 794                                                 bool true_last)
 795{
 796        int dir_in = hs_ep->dir_in;
 797        u32 mps = hs_ep->ep.maxpacket;
 798        u32 maxsize = 0;
 799        u32 offset = 0;
 800        u32 mask = 0;
 801        int i;
 802
 803        maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
 804
 805        hs_ep->desc_count = (len / maxsize) +
 806                                ((len % maxsize) ? 1 : 0);
 807        if (len == 0)
 808                hs_ep->desc_count = 1;
 809
 810        for (i = 0; i < hs_ep->desc_count; ++i) {
 811                (*desc)->status = 0;
 812                (*desc)->status |= (DEV_DMA_BUFF_STS_HBUSY
 813                                 << DEV_DMA_BUFF_STS_SHIFT);
 814
 815                if (len > maxsize) {
 816                        if (!hs_ep->index && !dir_in)
 817                                (*desc)->status |= (DEV_DMA_L | DEV_DMA_IOC);
 818
 819                        (*desc)->status |=
 820                                maxsize << DEV_DMA_NBYTES_SHIFT & mask;
 821                        (*desc)->buf = dma_buff + offset;
 822
 823                        len -= maxsize;
 824                        offset += maxsize;
 825                } else {
 826                        if (true_last)
 827                                (*desc)->status |= (DEV_DMA_L | DEV_DMA_IOC);
 828
 829                        if (dir_in)
 830                                (*desc)->status |= (len % mps) ? DEV_DMA_SHORT :
 831                                        ((hs_ep->send_zlp && true_last) ?
 832                                        DEV_DMA_SHORT : 0);
 833
 834                        (*desc)->status |=
 835                                len << DEV_DMA_NBYTES_SHIFT & mask;
 836                        (*desc)->buf = dma_buff + offset;
 837                }
 838
 839                (*desc)->status &= ~DEV_DMA_BUFF_STS_MASK;
 840                (*desc)->status |= (DEV_DMA_BUFF_STS_HREADY
 841                                 << DEV_DMA_BUFF_STS_SHIFT);
 842                (*desc)++;
 843        }
 844}
 845
 846/*
 847 * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain.
 848 * @hs_ep: The endpoint
 849 * @ureq: Request to transfer
 850 * @offset: offset in bytes
 851 * @len: Length of the transfer
 852 *
 853 * This function will iterate over descriptor chain and fill its entries
 854 * with corresponding information based on transfer data.
 855 */
 856static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep,
 857                                                 dma_addr_t dma_buff,
 858                                                 unsigned int len)
 859{
 860        struct usb_request *ureq = NULL;
 861        struct dwc2_dma_desc *desc = hs_ep->desc_list;
 862        struct scatterlist *sg;
 863        int i;
 864        u8 desc_count = 0;
 865
 866        if (hs_ep->req)
 867                ureq = &hs_ep->req->req;
 868
 869        /* non-DMA sg buffer */
 870        if (!ureq || !ureq->num_sgs) {
 871                dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc,
 872                        dma_buff, len, true);
 873                return;
 874        }
 875
 876        /* DMA sg buffer */
 877        for_each_sg(ureq->sg, sg, ureq->num_sgs, i) {
 878                dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc,
 879                        sg_dma_address(sg) + sg->offset, sg_dma_len(sg),
 880                        sg_is_last(sg));
 881                desc_count += hs_ep->desc_count;
 882        }
 883
 884        hs_ep->desc_count = desc_count;
 885}
 886
 887/*
 888 * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain.
 889 * @hs_ep: The isochronous endpoint.
 890 * @dma_buff: usb requests dma buffer.
 891 * @len: usb request transfer length.
 892 *
 893 * Fills next free descriptor with the data of the arrived usb request,
 894 * frame info, sets Last and IOC bits increments next_desc. If filled
 895 * descriptor is not the first one, removes L bit from the previous descriptor
 896 * status.
 897 */
 898static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
 899                                      dma_addr_t dma_buff, unsigned int len)
 900{
 901        struct dwc2_dma_desc *desc;
 902        struct dwc2_hsotg *hsotg = hs_ep->parent;
 903        u32 index;
 904        u32 mask = 0;
 905        u8 pid = 0;
 906
 907        dwc2_gadget_get_desc_params(hs_ep, &mask);
 908
 909        index = hs_ep->next_desc;
 910        desc = &hs_ep->desc_list[index];
 911
 912        /* Check if descriptor chain full */
 913        if ((desc->status >> DEV_DMA_BUFF_STS_SHIFT) ==
 914            DEV_DMA_BUFF_STS_HREADY) {
 915                dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__);
 916                return 1;
 917        }
 918
 919        /* Clear L bit of previous desc if more than one entries in the chain */
 920        if (hs_ep->next_desc)
 921                hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L;
 922
 923        dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n",
 924                __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index);
 925
 926        desc->status = 0;
 927        desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT);
 928
 929        desc->buf = dma_buff;
 930        desc->status |= (DEV_DMA_L | DEV_DMA_IOC |
 931                         ((len << DEV_DMA_NBYTES_SHIFT) & mask));
 932
 933        if (hs_ep->dir_in) {
 934                if (len)
 935                        pid = DIV_ROUND_UP(len, hs_ep->ep.maxpacket);
 936                else
 937                        pid = 1;
 938                desc->status |= ((pid << DEV_DMA_ISOC_PID_SHIFT) &
 939                                 DEV_DMA_ISOC_PID_MASK) |
 940                                ((len % hs_ep->ep.maxpacket) ?
 941                                 DEV_DMA_SHORT : 0) |
 942                                ((hs_ep->target_frame <<
 943                                  DEV_DMA_ISOC_FRNUM_SHIFT) &
 944                                 DEV_DMA_ISOC_FRNUM_MASK);
 945        }
 946
 947        desc->status &= ~DEV_DMA_BUFF_STS_MASK;
 948        desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT);
 949
 950        /* Increment frame number by interval for IN */
 951        if (hs_ep->dir_in)
 952                dwc2_gadget_incr_frame_num(hs_ep);
 953
 954        /* Update index of last configured entry in the chain */
 955        hs_ep->next_desc++;
 956        if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_HS_ISOC)
 957                hs_ep->next_desc = 0;
 958
 959        return 0;
 960}
 961
 962/*
 963 * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA
 964 * @hs_ep: The isochronous endpoint.
 965 *
 966 * Prepare descriptor chain for isochronous endpoints. Afterwards
 967 * write DMA address to HW and enable the endpoint.
 968 */
 969static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
 970{
 971        struct dwc2_hsotg *hsotg = hs_ep->parent;
 972        struct dwc2_hsotg_req *hs_req, *treq;
 973        int index = hs_ep->index;
 974        int ret;
 975        int i;
 976        u32 dma_reg;
 977        u32 depctl;
 978        u32 ctrl;
 979        struct dwc2_dma_desc *desc;
 980
 981        if (list_empty(&hs_ep->queue)) {
 982                hs_ep->target_frame = TARGET_FRAME_INITIAL;
 983                dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__);
 984                return;
 985        }
 986
 987        /* Initialize descriptor chain by Host Busy status */
 988        for (i = 0; i < MAX_DMA_DESC_NUM_HS_ISOC; i++) {
 989                desc = &hs_ep->desc_list[i];
 990                desc->status = 0;
 991                desc->status |= (DEV_DMA_BUFF_STS_HBUSY
 992                                    << DEV_DMA_BUFF_STS_SHIFT);
 993        }
 994
 995        hs_ep->next_desc = 0;
 996        list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) {
 997                dma_addr_t dma_addr = hs_req->req.dma;
 998
 999                if (hs_req->req.num_sgs) {
1000                        WARN_ON(hs_req->req.num_sgs > 1);
1001                        dma_addr = sg_dma_address(hs_req->req.sg);
1002                }
1003                ret = dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr,
1004                                                 hs_req->req.length);
1005                if (ret)
1006                        break;
1007        }
1008
1009        hs_ep->compl_desc = 0;
1010        depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
1011        dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
1012
1013        /* write descriptor chain address to control register */
1014        dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg);
1015
1016        ctrl = dwc2_readl(hsotg, depctl);
1017        ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
1018        dwc2_writel(hsotg, ctrl, depctl);
1019}
1020
1021/**
1022 * dwc2_hsotg_start_req - start a USB request from an endpoint's queue
1023 * @hsotg: The controller state.
1024 * @hs_ep: The endpoint to process a request for
1025 * @hs_req: The request to start.
1026 * @continuing: True if we are doing more for the current request.
1027 *
1028 * Start the given request running by setting the endpoint registers
1029 * appropriately, and writing any data to the FIFOs.
1030 */
1031static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
1032                                 struct dwc2_hsotg_ep *hs_ep,
1033                                struct dwc2_hsotg_req *hs_req,
1034                                bool continuing)
1035{
1036        struct usb_request *ureq = &hs_req->req;
1037        int index = hs_ep->index;
1038        int dir_in = hs_ep->dir_in;
1039        u32 epctrl_reg;
1040        u32 epsize_reg;
1041        u32 epsize;
1042        u32 ctrl;
1043        unsigned int length;
1044        unsigned int packets;
1045        unsigned int maxreq;
1046        unsigned int dma_reg;
1047
1048        if (index != 0) {
1049                if (hs_ep->req && !continuing) {
1050                        dev_err(hsotg->dev, "%s: active request\n", __func__);
1051                        WARN_ON(1);
1052                        return;
1053                } else if (hs_ep->req != hs_req && continuing) {
1054                        dev_err(hsotg->dev,
1055                                "%s: continue different req\n", __func__);
1056                        WARN_ON(1);
1057                        return;
1058                }
1059        }
1060
1061        dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
1062        epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
1063        epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
1064
1065        dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x, ep %d, dir %s\n",
1066                __func__, dwc2_readl(hsotg, epctrl_reg), index,
1067                hs_ep->dir_in ? "in" : "out");
1068
1069        /* If endpoint is stalled, we will restart request later */
1070        ctrl = dwc2_readl(hsotg, epctrl_reg);
1071
1072        if (index && ctrl & DXEPCTL_STALL) {
1073                dev_warn(hsotg->dev, "%s: ep%d is stalled\n", __func__, index);
1074                return;
1075        }
1076
1077        length = ureq->length - ureq->actual;
1078        dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n",
1079                ureq->length, ureq->actual);
1080
1081        if (!using_desc_dma(hsotg))
1082                maxreq = get_ep_limit(hs_ep);
1083        else
1084                maxreq = dwc2_gadget_get_chain_limit(hs_ep);
1085
1086        if (length > maxreq) {
1087                int round = maxreq % hs_ep->ep.maxpacket;
1088
1089                dev_dbg(hsotg->dev, "%s: length %d, max-req %d, r %d\n",
1090                        __func__, length, maxreq, round);
1091
1092                /* round down to multiple of packets */
1093                if (round)
1094                        maxreq -= round;
1095
1096                length = maxreq;
1097        }
1098
1099        if (length)
1100                packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);
1101        else
1102                packets = 1;    /* send one packet if length is zero. */
1103
1104        if (dir_in && index != 0)
1105                if (hs_ep->isochronous)
1106                        epsize = DXEPTSIZ_MC(packets);
1107                else
1108                        epsize = DXEPTSIZ_MC(1);
1109        else
1110                epsize = 0;
1111
1112        /*
1113         * zero length packet should be programmed on its own and should not
1114         * be counted in DIEPTSIZ.PktCnt with other packets.
1115         */
1116        if (dir_in && ureq->zero && !continuing) {
1117                /* Test if zlp is actually required. */
1118                if ((ureq->length >= hs_ep->ep.maxpacket) &&
1119                    !(ureq->length % hs_ep->ep.maxpacket))
1120                        hs_ep->send_zlp = 1;
1121        }
1122
1123        epsize |= DXEPTSIZ_PKTCNT(packets);
1124        epsize |= DXEPTSIZ_XFERSIZE(length);
1125
1126        dev_dbg(hsotg->dev, "%s: %d@%d/%d, 0x%08x => 0x%08x\n",
1127                __func__, packets, length, ureq->length, epsize, epsize_reg);
1128
1129        /* store the request as the current one we're doing */
1130        hs_ep->req = hs_req;
1131
1132        if (using_desc_dma(hsotg)) {
1133                u32 offset = 0;
1134                u32 mps = hs_ep->ep.maxpacket;
1135
1136                /* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */
1137                if (!dir_in) {
1138                        if (!index)
1139                                length = mps;
1140                        else if (length % mps)
1141                                length += (mps - (length % mps));
1142                }
1143
1144                if (continuing)
1145                        offset = ureq->actual;
1146
1147                /* Fill DDMA chain entries */
1148                dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset,
1149                                                     length);
1150
1151                /* write descriptor chain address to control register */
1152                dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg);
1153
1154                dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n",
1155                        __func__, (u32)hs_ep->desc_list_dma, dma_reg);
1156        } else {
1157                /* write size / packets */
1158                dwc2_writel(hsotg, epsize, epsize_reg);
1159
1160                if (using_dma(hsotg) && !continuing && (length != 0)) {
1161                        /*
1162                         * write DMA address to control register, buffer
1163                         * already synced by dwc2_hsotg_ep_queue().
1164                         */
1165
1166                        dwc2_writel(hsotg, ureq->dma, dma_reg);
1167
1168                        dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
1169                                __func__, &ureq->dma, dma_reg);
1170                }
1171        }
1172
1173        if (hs_ep->isochronous && hs_ep->interval == 1) {
1174                hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
1175                dwc2_gadget_incr_frame_num(hs_ep);
1176
1177                if (hs_ep->target_frame & 0x1)
1178                        ctrl |= DXEPCTL_SETODDFR;
1179                else
1180                        ctrl |= DXEPCTL_SETEVENFR;
1181        }
1182
1183        ctrl |= DXEPCTL_EPENA;  /* ensure ep enabled */
1184
1185        dev_dbg(hsotg->dev, "ep0 state:%d\n", hsotg->ep0_state);
1186
1187        /* For Setup request do not clear NAK */
1188        if (!(index == 0 && hsotg->ep0_state == DWC2_EP0_SETUP))
1189                ctrl |= DXEPCTL_CNAK;   /* clear NAK set by core */
1190
1191        dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
1192        dwc2_writel(hsotg, ctrl, epctrl_reg);
1193
1194        /*
1195         * set these, it seems that DMA support increments past the end
1196         * of the packet buffer so we need to calculate the length from
1197         * this information.
1198         */
1199        hs_ep->size_loaded = length;
1200        hs_ep->last_load = ureq->actual;
1201
1202        if (dir_in && !using_dma(hsotg)) {
1203                /* set these anyway, we may need them for non-periodic in */
1204                hs_ep->fifo_load = 0;
1205
1206                dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
1207        }
1208
1209        /*
1210         * Note, trying to clear the NAK here causes problems with transmit
1211         * on the S3C6400 ending up with the TXFIFO becoming full.
1212         */
1213
1214        /* check ep is enabled */
1215        if (!(dwc2_readl(hsotg, epctrl_reg) & DXEPCTL_EPENA))
1216                dev_dbg(hsotg->dev,
1217                        "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n",
1218                         index, dwc2_readl(hsotg, epctrl_reg));
1219
1220        dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n",
1221                __func__, dwc2_readl(hsotg, epctrl_reg));
1222
1223        /* enable ep interrupts */
1224        dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);
1225}
1226
1227/**
1228 * dwc2_hsotg_map_dma - map the DMA memory being used for the request
1229 * @hsotg: The device state.
1230 * @hs_ep: The endpoint the request is on.
1231 * @req: The request being processed.
1232 *
1233 * We've been asked to queue a request, so ensure that the memory buffer
1234 * is correctly setup for DMA. If we've been passed an extant DMA address
1235 * then ensure the buffer has been synced to memory. If our buffer has no
1236 * DMA memory, then we map the memory and mark our request to allow us to
1237 * cleanup on completion.
1238 */
1239static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg,
1240                              struct dwc2_hsotg_ep *hs_ep,
1241                             struct usb_request *req)
1242{
1243        int ret;
1244
1245        hs_ep->map_dir = hs_ep->dir_in;
1246        ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);
1247        if (ret)
1248                goto dma_error;
1249
1250        return 0;
1251
1252dma_error:
1253        dev_err(hsotg->dev, "%s: failed to map buffer %p, %d bytes\n",
1254                __func__, req->buf, req->length);
1255
1256        return -EIO;
1257}
1258
1259static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg,
1260                                                 struct dwc2_hsotg_ep *hs_ep,
1261                                                 struct dwc2_hsotg_req *hs_req)
1262{
1263        void *req_buf = hs_req->req.buf;
1264
1265        /* If dma is not being used or buffer is aligned */
1266        if (!using_dma(hsotg) || !((long)req_buf & 3))
1267                return 0;
1268
1269        WARN_ON(hs_req->saved_req_buf);
1270
1271        dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__,
1272                hs_ep->ep.name, req_buf, hs_req->req.length);
1273
1274        hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC);
1275        if (!hs_req->req.buf) {
1276                hs_req->req.buf = req_buf;
1277                dev_err(hsotg->dev,
1278                        "%s: unable to allocate memory for bounce buffer\n",
1279                        __func__);
1280                return -ENOMEM;
1281        }
1282
1283        /* Save actual buffer */
1284        hs_req->saved_req_buf = req_buf;
1285
1286        if (hs_ep->dir_in)
1287                memcpy(hs_req->req.buf, req_buf, hs_req->req.length);
1288        return 0;
1289}
1290
1291static void
1292dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg,
1293                                         struct dwc2_hsotg_ep *hs_ep,
1294                                         struct dwc2_hsotg_req *hs_req)
1295{
1296        /* If dma is not being used or buffer was aligned */
1297        if (!using_dma(hsotg) || !hs_req->saved_req_buf)
1298                return;
1299
1300        dev_dbg(hsotg->dev, "%s: %s: status=%d actual-length=%d\n", __func__,
1301                hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);
1302
1303        /* Copy data from bounce buffer on successful out transfer */
1304        if (!hs_ep->dir_in && !hs_req->req.status)
1305                memcpy(hs_req->saved_req_buf, hs_req->req.buf,
1306                       hs_req->req.actual);
1307
1308        /* Free bounce buffer */
1309        kfree(hs_req->req.buf);
1310
1311        hs_req->req.buf = hs_req->saved_req_buf;
1312        hs_req->saved_req_buf = NULL;
1313}
1314
1315/**
1316 * dwc2_gadget_target_frame_elapsed - Checks target frame
1317 * @hs_ep: The driver endpoint to check
1318 *
1319 * Returns 1 if targeted frame elapsed. If returned 1 then we need to drop
1320 * corresponding transfer.
1321 */
1322static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep)
1323{
1324        struct dwc2_hsotg *hsotg = hs_ep->parent;
1325        u32 target_frame = hs_ep->target_frame;
1326        u32 current_frame = hsotg->frame_number;
1327        bool frame_overrun = hs_ep->frame_overrun;
1328
1329        if (!frame_overrun && current_frame >= target_frame)
1330                return true;
1331
1332        if (frame_overrun && current_frame >= target_frame &&
1333            ((current_frame - target_frame) < DSTS_SOFFN_LIMIT / 2))
1334                return true;
1335
1336        return false;
1337}
1338
1339/*
1340 * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers
1341 * @hsotg: The driver state
1342 * @hs_ep: the ep descriptor chain is for
1343 *
1344 * Called to update EP0 structure's pointers depend on stage of
1345 * control transfer.
1346 */
1347static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg,
1348                                          struct dwc2_hsotg_ep *hs_ep)
1349{
1350        switch (hsotg->ep0_state) {
1351        case DWC2_EP0_SETUP:
1352        case DWC2_EP0_STATUS_OUT:
1353                hs_ep->desc_list = hsotg->setup_desc[0];
1354                hs_ep->desc_list_dma = hsotg->setup_desc_dma[0];
1355                break;
1356        case DWC2_EP0_DATA_IN:
1357        case DWC2_EP0_STATUS_IN:
1358                hs_ep->desc_list = hsotg->ctrl_in_desc;
1359                hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma;
1360                break;
1361        case DWC2_EP0_DATA_OUT:
1362                hs_ep->desc_list = hsotg->ctrl_out_desc;
1363                hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma;
1364                break;
1365        default:
1366                dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n",
1367                        hsotg->ep0_state);
1368                return -EINVAL;
1369        }
1370
1371        return 0;
1372}
1373
1374static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
1375                               gfp_t gfp_flags)
1376{
1377        struct dwc2_hsotg_req *hs_req = our_req(req);
1378        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1379        struct dwc2_hsotg *hs = hs_ep->parent;
1380        bool first;
1381        int ret;
1382        u32 maxsize = 0;
1383        u32 mask = 0;
1384
1385
1386        dev_dbg(hs->dev, "%s: req %p: %d@%p, noi=%d, zero=%d, snok=%d\n",
1387                ep->name, req, req->length, req->buf, req->no_interrupt,
1388                req->zero, req->short_not_ok);
1389
1390        /* Prevent new request submission when controller is suspended */
1391        if (hs->lx_state != DWC2_L0) {
1392                dev_dbg(hs->dev, "%s: submit request only in active state\n",
1393                        __func__);
1394                return -EAGAIN;
1395        }
1396
1397        /* initialise status of the request */
1398        INIT_LIST_HEAD(&hs_req->queue);
1399        req->actual = 0;
1400        req->status = -EINPROGRESS;
1401
1402        /* Don't queue ISOC request if length greater than mps*mc */
1403        if (hs_ep->isochronous &&
1404            req->length > (hs_ep->mc * hs_ep->ep.maxpacket)) {
1405                dev_err(hs->dev, "req length > maxpacket*mc\n");
1406                return -EINVAL;
1407        }
1408
1409        /* In DDMA mode for ISOC's don't queue request if length greater
1410         * than descriptor limits.
1411         */
1412        if (using_desc_dma(hs) && hs_ep->isochronous) {
1413                maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
1414                if (hs_ep->dir_in && req->length > maxsize) {
1415                        dev_err(hs->dev, "wrong length %d (maxsize=%d)\n",
1416                                req->length, maxsize);
1417                        return -EINVAL;
1418                }
1419
1420                if (!hs_ep->dir_in && req->length > hs_ep->ep.maxpacket) {
1421                        dev_err(hs->dev, "ISOC OUT: wrong length %d (mps=%d)\n",
1422                                req->length, hs_ep->ep.maxpacket);
1423                        return -EINVAL;
1424                }
1425        }
1426
1427        ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);
1428        if (ret)
1429                return ret;
1430
1431        /* if we're using DMA, sync the buffers as necessary */
1432        if (using_dma(hs)) {
1433                ret = dwc2_hsotg_map_dma(hs, hs_ep, req);
1434                if (ret)
1435                        return ret;
1436        }
1437        /* If using descriptor DMA configure EP0 descriptor chain pointers */
1438        if (using_desc_dma(hs) && !hs_ep->index) {
1439                ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep);
1440                if (ret)
1441                        return ret;
1442        }
1443
1444        first = list_empty(&hs_ep->queue);
1445        list_add_tail(&hs_req->queue, &hs_ep->queue);
1446
1447        /*
1448         * Handle DDMA isochronous transfers separately - just add new entry
1449         * to the descriptor chain.
1450         * Transfer will be started once SW gets either one of NAK or
1451         * OutTknEpDis interrupts.
1452         */
1453        if (using_desc_dma(hs) && hs_ep->isochronous) {
1454                if (hs_ep->target_frame != TARGET_FRAME_INITIAL) {
1455                        dma_addr_t dma_addr = hs_req->req.dma;
1456
1457                        if (hs_req->req.num_sgs) {
1458                                WARN_ON(hs_req->req.num_sgs > 1);
1459                                dma_addr = sg_dma_address(hs_req->req.sg);
1460                        }
1461                        dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr,
1462                                                   hs_req->req.length);
1463                }
1464                return 0;
1465        }
1466
1467        /* Change EP direction if status phase request is after data out */
1468        if (!hs_ep->index && !req->length && !hs_ep->dir_in &&
1469            hs->ep0_state == DWC2_EP0_DATA_OUT)
1470                hs_ep->dir_in = 1;
1471
1472        if (first) {
1473                if (!hs_ep->isochronous) {
1474                        dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1475                        return 0;
1476                }
1477
1478                /* Update current frame number value. */
1479                hs->frame_number = dwc2_hsotg_read_frameno(hs);
1480                while (dwc2_gadget_target_frame_elapsed(hs_ep)) {
1481                        dwc2_gadget_incr_frame_num(hs_ep);
1482                        /* Update current frame number value once more as it
1483                         * changes here.
1484                         */
1485                        hs->frame_number = dwc2_hsotg_read_frameno(hs);
1486                }
1487
1488                if (hs_ep->target_frame != TARGET_FRAME_INITIAL)
1489                        dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1490        }
1491        return 0;
1492}
1493
1494static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req,
1495                                    gfp_t gfp_flags)
1496{
1497        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1498        struct dwc2_hsotg *hs = hs_ep->parent;
1499        unsigned long flags;
1500        int ret;
1501
1502        spin_lock_irqsave(&hs->lock, flags);
1503        ret = dwc2_hsotg_ep_queue(ep, req, gfp_flags);
1504        spin_unlock_irqrestore(&hs->lock, flags);
1505
1506        return ret;
1507}
1508
1509static void dwc2_hsotg_ep_free_request(struct usb_ep *ep,
1510                                       struct usb_request *req)
1511{
1512        struct dwc2_hsotg_req *hs_req = our_req(req);
1513
1514        kfree(hs_req);
1515}
1516
1517/**
1518 * dwc2_hsotg_complete_oursetup - setup completion callback
1519 * @ep: The endpoint the request was on.
1520 * @req: The request completed.
1521 *
1522 * Called on completion of any requests the driver itself
1523 * submitted that need cleaning up.
1524 */
1525static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep,
1526                                         struct usb_request *req)
1527{
1528        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1529        struct dwc2_hsotg *hsotg = hs_ep->parent;
1530
1531        dev_dbg(hsotg->dev, "%s: ep %p, req %p\n", __func__, ep, req);
1532
1533        dwc2_hsotg_ep_free_request(ep, req);
1534}
1535
1536/**
1537 * ep_from_windex - convert control wIndex value to endpoint
1538 * @hsotg: The driver state.
1539 * @windex: The control request wIndex field (in host order).
1540 *
1541 * Convert the given wIndex into a pointer to an driver endpoint
1542 * structure, or return NULL if it is not a valid endpoint.
1543 */
1544static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg,
1545                                            u32 windex)
1546{
1547        int dir = (windex & USB_DIR_IN) ? 1 : 0;
1548        int idx = windex & 0x7F;
1549
1550        if (windex >= 0x100)
1551                return NULL;
1552
1553        if (idx > hsotg->num_of_eps)
1554                return NULL;
1555
1556        return index_to_ep(hsotg, idx, dir);
1557}
1558
1559/**
1560 * dwc2_hsotg_set_test_mode - Enable usb Test Modes
1561 * @hsotg: The driver state.
1562 * @testmode: requested usb test mode
1563 * Enable usb Test Mode requested by the Host.
1564 */
1565int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode)
1566{
1567        int dctl = dwc2_readl(hsotg, DCTL);
1568
1569        dctl &= ~DCTL_TSTCTL_MASK;
1570        switch (testmode) {
1571        case USB_TEST_J:
1572        case USB_TEST_K:
1573        case USB_TEST_SE0_NAK:
1574        case USB_TEST_PACKET:
1575        case USB_TEST_FORCE_ENABLE:
1576                dctl |= testmode << DCTL_TSTCTL_SHIFT;
1577                break;
1578        default:
1579                return -EINVAL;
1580        }
1581        dwc2_writel(hsotg, dctl, DCTL);
1582        return 0;
1583}
1584
1585/**
1586 * dwc2_hsotg_send_reply - send reply to control request
1587 * @hsotg: The device state
1588 * @ep: Endpoint 0
1589 * @buff: Buffer for request
1590 * @length: Length of reply.
1591 *
1592 * Create a request and queue it on the given endpoint. This is useful as
1593 * an internal method of sending replies to certain control requests, etc.
1594 */
1595static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg,
1596                                 struct dwc2_hsotg_ep *ep,
1597                                void *buff,
1598                                int length)
1599{
1600        struct usb_request *req;
1601        int ret;
1602
1603        dev_dbg(hsotg->dev, "%s: buff %p, len %d\n", __func__, buff, length);
1604
1605        req = dwc2_hsotg_ep_alloc_request(&ep->ep, GFP_ATOMIC);
1606        hsotg->ep0_reply = req;
1607        if (!req) {
1608                dev_warn(hsotg->dev, "%s: cannot alloc req\n", __func__);
1609                return -ENOMEM;
1610        }
1611
1612        req->buf = hsotg->ep0_buff;
1613        req->length = length;
1614        /*
1615         * zero flag is for sending zlp in DATA IN stage. It has no impact on
1616         * STATUS stage.
1617         */
1618        req->zero = 0;
1619        req->complete = dwc2_hsotg_complete_oursetup;
1620
1621        if (length)
1622                memcpy(req->buf, buff, length);
1623
1624        ret = dwc2_hsotg_ep_queue(&ep->ep, req, GFP_ATOMIC);
1625        if (ret) {
1626                dev_warn(hsotg->dev, "%s: cannot queue req\n", __func__);
1627                return ret;
1628        }
1629
1630        return 0;
1631}
1632
1633/**
1634 * dwc2_hsotg_process_req_status - process request GET_STATUS
1635 * @hsotg: The device state
1636 * @ctrl: USB control request
1637 */
1638static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg,
1639                                         struct usb_ctrlrequest *ctrl)
1640{
1641        struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1642        struct dwc2_hsotg_ep *ep;
1643        __le16 reply;
1644        u16 status;
1645        int ret;
1646
1647        dev_dbg(hsotg->dev, "%s: USB_REQ_GET_STATUS\n", __func__);
1648
1649        if (!ep0->dir_in) {
1650                dev_warn(hsotg->dev, "%s: direction out?\n", __func__);
1651                return -EINVAL;
1652        }
1653
1654        switch (ctrl->bRequestType & USB_RECIP_MASK) {
1655        case USB_RECIP_DEVICE:
1656                status = hsotg->gadget.is_selfpowered <<
1657                         USB_DEVICE_SELF_POWERED;
1658                status |= hsotg->remote_wakeup_allowed <<
1659                          USB_DEVICE_REMOTE_WAKEUP;
1660                reply = cpu_to_le16(status);
1661                break;
1662
1663        case USB_RECIP_INTERFACE:
1664                /* currently, the data result should be zero */
1665                reply = cpu_to_le16(0);
1666                break;
1667
1668        case USB_RECIP_ENDPOINT:
1669                ep = ep_from_windex(hsotg, le16_to_cpu(ctrl->wIndex));
1670                if (!ep)
1671                        return -ENOENT;
1672
1673                reply = cpu_to_le16(ep->halted ? 1 : 0);
1674                break;
1675
1676        default:
1677                return 0;
1678        }
1679
1680        if (le16_to_cpu(ctrl->wLength) != 2)
1681                return -EINVAL;
1682
1683        ret = dwc2_hsotg_send_reply(hsotg, ep0, &reply, 2);
1684        if (ret) {
1685                dev_err(hsotg->dev, "%s: failed to send reply\n", __func__);
1686                return ret;
1687        }
1688
1689        return 1;
1690}
1691
1692static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now);
1693
1694/**
1695 * get_ep_head - return the first request on the endpoint
1696 * @hs_ep: The controller endpoint to get
1697 *
1698 * Get the first request on the endpoint.
1699 */
1700static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)
1701{
1702        return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req,
1703                                        queue);
1704}
1705
1706/**
1707 * dwc2_gadget_start_next_request - Starts next request from ep queue
1708 * @hs_ep: Endpoint structure
1709 *
1710 * If queue is empty and EP is ISOC-OUT - unmasks OUTTKNEPDIS which is masked
1711 * in its handler. Hence we need to unmask it here to be able to do
1712 * resynchronization.
1713 */
1714static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep)
1715{
1716        u32 mask;
1717        struct dwc2_hsotg *hsotg = hs_ep->parent;
1718        int dir_in = hs_ep->dir_in;
1719        struct dwc2_hsotg_req *hs_req;
1720        u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
1721
1722        if (!list_empty(&hs_ep->queue)) {
1723                hs_req = get_ep_head(hs_ep);
1724                dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false);
1725                return;
1726        }
1727        if (!hs_ep->isochronous)
1728                return;
1729
1730        if (dir_in) {
1731                dev_dbg(hsotg->dev, "%s: No more ISOC-IN requests\n",
1732                        __func__);
1733        } else {
1734                dev_dbg(hsotg->dev, "%s: No more ISOC-OUT requests\n",
1735                        __func__);
1736                mask = dwc2_readl(hsotg, epmsk_reg);
1737                mask |= DOEPMSK_OUTTKNEPDISMSK;
1738                dwc2_writel(hsotg, mask, epmsk_reg);
1739        }
1740}
1741
1742/**
1743 * dwc2_hsotg_process_req_feature - process request {SET,CLEAR}_FEATURE
1744 * @hsotg: The device state
1745 * @ctrl: USB control request
1746 */
1747static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
1748                                          struct usb_ctrlrequest *ctrl)
1749{
1750        struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1751        struct dwc2_hsotg_req *hs_req;
1752        bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
1753        struct dwc2_hsotg_ep *ep;
1754        int ret;
1755        bool halted;
1756        u32 recip;
1757        u32 wValue;
1758        u32 wIndex;
1759
1760        dev_dbg(hsotg->dev, "%s: %s_FEATURE\n",
1761                __func__, set ? "SET" : "CLEAR");
1762
1763        wValue = le16_to_cpu(ctrl->wValue);
1764        wIndex = le16_to_cpu(ctrl->wIndex);
1765        recip = ctrl->bRequestType & USB_RECIP_MASK;
1766
1767        switch (recip) {
1768        case USB_RECIP_DEVICE:
1769                switch (wValue) {
1770                case USB_DEVICE_REMOTE_WAKEUP:
1771                        if (set)
1772                                hsotg->remote_wakeup_allowed = 1;
1773                        else
1774                                hsotg->remote_wakeup_allowed = 0;
1775                        break;
1776
1777                case USB_DEVICE_TEST_MODE:
1778                        if ((wIndex & 0xff) != 0)
1779                                return -EINVAL;
1780                        if (!set)
1781                                return -EINVAL;
1782
1783                        hsotg->test_mode = wIndex >> 8;
1784                        break;
1785                default:
1786                        return -ENOENT;
1787                }
1788
1789                ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1790                if (ret) {
1791                        dev_err(hsotg->dev,
1792                                "%s: failed to send reply\n", __func__);
1793                        return ret;
1794                }
1795                break;
1796
1797        case USB_RECIP_ENDPOINT:
1798                ep = ep_from_windex(hsotg, wIndex);
1799                if (!ep) {
1800                        dev_dbg(hsotg->dev, "%s: no endpoint for 0x%04x\n",
1801                                __func__, wIndex);
1802                        return -ENOENT;
1803                }
1804
1805                switch (wValue) {
1806                case USB_ENDPOINT_HALT:
1807                        halted = ep->halted;
1808
1809                        dwc2_hsotg_ep_sethalt(&ep->ep, set, true);
1810
1811                        ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1812                        if (ret) {
1813                                dev_err(hsotg->dev,
1814                                        "%s: failed to send reply\n", __func__);
1815                                return ret;
1816                        }
1817
1818                        /*
1819                         * we have to complete all requests for ep if it was
1820                         * halted, and the halt was cleared by CLEAR_FEATURE
1821                         */
1822
1823                        if (!set && halted) {
1824                                /*
1825                                 * If we have request in progress,
1826                                 * then complete it
1827                                 */
1828                                if (ep->req) {
1829                                        hs_req = ep->req;
1830                                        ep->req = NULL;
1831                                        list_del_init(&hs_req->queue);
1832                                        if (hs_req->req.complete) {
1833                                                spin_unlock(&hsotg->lock);
1834                                                usb_gadget_giveback_request(
1835                                                        &ep->ep, &hs_req->req);
1836                                                spin_lock(&hsotg->lock);
1837                                        }
1838                                }
1839
1840                                /* If we have pending request, then start it */
1841                                if (!ep->req)
1842                                        dwc2_gadget_start_next_request(ep);
1843                        }
1844
1845                        break;
1846
1847                default:
1848                        return -ENOENT;
1849                }
1850                break;
1851        default:
1852                return -ENOENT;
1853        }
1854        return 1;
1855}
1856
1857static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg);
1858
1859/**
1860 * dwc2_hsotg_stall_ep0 - stall ep0
1861 * @hsotg: The device state
1862 *
1863 * Set stall for ep0 as response for setup request.
1864 */
1865static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg)
1866{
1867        struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1868        u32 reg;
1869        u32 ctrl;
1870
1871        dev_dbg(hsotg->dev, "ep0 stall (dir=%d)\n", ep0->dir_in);
1872        reg = (ep0->dir_in) ? DIEPCTL0 : DOEPCTL0;
1873
1874        /*
1875         * DxEPCTL_Stall will be cleared by EP once it has
1876         * taken effect, so no need to clear later.
1877         */
1878
1879        ctrl = dwc2_readl(hsotg, reg);
1880        ctrl |= DXEPCTL_STALL;
1881        ctrl |= DXEPCTL_CNAK;
1882        dwc2_writel(hsotg, ctrl, reg);
1883
1884        dev_dbg(hsotg->dev,
1885                "written DXEPCTL=0x%08x to %08x (DXEPCTL=0x%08x)\n",
1886                ctrl, reg, dwc2_readl(hsotg, reg));
1887
1888         /*
1889          * complete won't be called, so we enqueue
1890          * setup request here
1891          */
1892         dwc2_hsotg_enqueue_setup(hsotg);
1893}
1894
1895/**
1896 * dwc2_hsotg_process_control - process a control request
1897 * @hsotg: The device state
1898 * @ctrl: The control request received
1899 *
1900 * The controller has received the SETUP phase of a control request, and
1901 * needs to work out what to do next (and whether to pass it on to the
1902 * gadget driver).
1903 */
1904static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg,
1905                                       struct usb_ctrlrequest *ctrl)
1906{
1907        struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1908        int ret = 0;
1909        u32 dcfg;
1910
1911        dev_dbg(hsotg->dev,
1912                "ctrl Type=%02x, Req=%02x, V=%04x, I=%04x, L=%04x\n",
1913                ctrl->bRequestType, ctrl->bRequest, ctrl->wValue,
1914                ctrl->wIndex, ctrl->wLength);
1915
1916        if (ctrl->wLength == 0) {
1917                ep0->dir_in = 1;
1918                hsotg->ep0_state = DWC2_EP0_STATUS_IN;
1919        } else if (ctrl->bRequestType & USB_DIR_IN) {
1920                ep0->dir_in = 1;
1921                hsotg->ep0_state = DWC2_EP0_DATA_IN;
1922        } else {
1923                ep0->dir_in = 0;
1924                hsotg->ep0_state = DWC2_EP0_DATA_OUT;
1925        }
1926
1927        if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1928                switch (ctrl->bRequest) {
1929                case USB_REQ_SET_ADDRESS:
1930                        hsotg->connected = 1;
1931                        dcfg = dwc2_readl(hsotg, DCFG);
1932                        dcfg &= ~DCFG_DEVADDR_MASK;
1933                        dcfg |= (le16_to_cpu(ctrl->wValue) <<
1934                                 DCFG_DEVADDR_SHIFT) & DCFG_DEVADDR_MASK;
1935                        dwc2_writel(hsotg, dcfg, DCFG);
1936
1937                        dev_info(hsotg->dev, "new address %d\n", ctrl->wValue);
1938
1939                        ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1940                        return;
1941
1942                case USB_REQ_GET_STATUS:
1943                        ret = dwc2_hsotg_process_req_status(hsotg, ctrl);
1944                        break;
1945
1946                case USB_REQ_CLEAR_FEATURE:
1947                case USB_REQ_SET_FEATURE:
1948                        ret = dwc2_hsotg_process_req_feature(hsotg, ctrl);
1949                        break;
1950                }
1951        }
1952
1953        /* as a fallback, try delivering it to the driver to deal with */
1954
1955        if (ret == 0 && hsotg->driver) {
1956                spin_unlock(&hsotg->lock);
1957                ret = hsotg->driver->setup(&hsotg->gadget, ctrl);
1958                spin_lock(&hsotg->lock);
1959                if (ret < 0)
1960                        dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret);
1961        }
1962
1963        hsotg->delayed_status = false;
1964        if (ret == USB_GADGET_DELAYED_STATUS)
1965                hsotg->delayed_status = true;
1966
1967        /*
1968         * the request is either unhandlable, or is not formatted correctly
1969         * so respond with a STALL for the status stage to indicate failure.
1970         */
1971
1972        if (ret < 0)
1973                dwc2_hsotg_stall_ep0(hsotg);
1974}
1975
1976/**
1977 * dwc2_hsotg_complete_setup - completion of a setup transfer
1978 * @ep: The endpoint the request was on.
1979 * @req: The request completed.
1980 *
1981 * Called on completion of any requests the driver itself submitted for
1982 * EP0 setup packets
1983 */
1984static void dwc2_hsotg_complete_setup(struct usb_ep *ep,
1985                                      struct usb_request *req)
1986{
1987        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1988        struct dwc2_hsotg *hsotg = hs_ep->parent;
1989
1990        if (req->status < 0) {
1991                dev_dbg(hsotg->dev, "%s: failed %d\n", __func__, req->status);
1992                return;
1993        }
1994
1995        spin_lock(&hsotg->lock);
1996        if (req->actual == 0)
1997                dwc2_hsotg_enqueue_setup(hsotg);
1998        else
1999                dwc2_hsotg_process_control(hsotg, req->buf);
2000        spin_unlock(&hsotg->lock);
2001}
2002
2003/**
2004 * dwc2_hsotg_enqueue_setup - start a request for EP0 packets
2005 * @hsotg: The device state.
2006 *
2007 * Enqueue a request on EP0 if necessary to received any SETUP packets
2008 * received from the host.
2009 */
2010static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg)
2011{
2012        struct usb_request *req = hsotg->ctrl_req;
2013        struct dwc2_hsotg_req *hs_req = our_req(req);
2014        int ret;
2015
2016        dev_dbg(hsotg->dev, "%s: queueing setup request\n", __func__);
2017
2018        req->zero = 0;
2019        req->length = 8;
2020        req->buf = hsotg->ctrl_buff;
2021        req->complete = dwc2_hsotg_complete_setup;
2022
2023        if (!list_empty(&hs_req->queue)) {
2024                dev_dbg(hsotg->dev, "%s already queued???\n", __func__);
2025                return;
2026        }
2027
2028        hsotg->eps_out[0]->dir_in = 0;
2029        hsotg->eps_out[0]->send_zlp = 0;
2030        hsotg->ep0_state = DWC2_EP0_SETUP;
2031
2032        ret = dwc2_hsotg_ep_queue(&hsotg->eps_out[0]->ep, req, GFP_ATOMIC);
2033        if (ret < 0) {
2034                dev_err(hsotg->dev, "%s: failed queue (%d)\n", __func__, ret);
2035                /*
2036                 * Don't think there's much we can do other than watch the
2037                 * driver fail.
2038                 */
2039        }
2040}
2041
2042static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg,
2043                                   struct dwc2_hsotg_ep *hs_ep)
2044{
2045        u32 ctrl;
2046        u8 index = hs_ep->index;
2047        u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
2048        u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
2049
2050        if (hs_ep->dir_in)
2051                dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n",
2052                        index);
2053        else
2054                dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n",
2055                        index);
2056        if (using_desc_dma(hsotg)) {
2057                /* Not specific buffer needed for ep0 ZLP */
2058                dma_addr_t dma = hs_ep->desc_list_dma;
2059
2060                if (!index)
2061                        dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep);
2062
2063                dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0);
2064        } else {
2065                dwc2_writel(hsotg, DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
2066                            DXEPTSIZ_XFERSIZE(0),
2067                            epsiz_reg);
2068        }
2069
2070        ctrl = dwc2_readl(hsotg, epctl_reg);
2071        ctrl |= DXEPCTL_CNAK;  /* clear NAK set by core */
2072        ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */
2073        ctrl |= DXEPCTL_USBACTEP;
2074        dwc2_writel(hsotg, ctrl, epctl_reg);
2075}
2076
2077/**
2078 * dwc2_hsotg_complete_request - complete a request given to us
2079 * @hsotg: The device state.
2080 * @hs_ep: The endpoint the request was on.
2081 * @hs_req: The request to complete.
2082 * @result: The result code (0 => Ok, otherwise errno)
2083 *
2084 * The given request has finished, so call the necessary completion
2085 * if it has one and then look to see if we can start a new request
2086 * on the endpoint.
2087 *
2088 * Note, expects the ep to already be locked as appropriate.
2089 */
2090static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
2091                                        struct dwc2_hsotg_ep *hs_ep,
2092                                       struct dwc2_hsotg_req *hs_req,
2093                                       int result)
2094{
2095        if (!hs_req) {
2096                dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__);
2097                return;
2098        }
2099
2100        dev_dbg(hsotg->dev, "complete: ep %p %s, req %p, %d => %p\n",
2101                hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);
2102
2103        /*
2104         * only replace the status if we've not already set an error
2105         * from a previous transaction
2106         */
2107
2108        if (hs_req->req.status == -EINPROGRESS)
2109                hs_req->req.status = result;
2110
2111        if (using_dma(hsotg))
2112                dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req);
2113
2114        dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);
2115
2116        hs_ep->req = NULL;
2117        list_del_init(&hs_req->queue);
2118
2119        /*
2120         * call the complete request with the locks off, just in case the
2121         * request tries to queue more work for this endpoint.
2122         */
2123
2124        if (hs_req->req.complete) {
2125                spin_unlock(&hsotg->lock);
2126                usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);
2127                spin_lock(&hsotg->lock);
2128        }
2129
2130        /* In DDMA don't need to proceed to starting of next ISOC request */
2131        if (using_desc_dma(hsotg) && hs_ep->isochronous)
2132                return;
2133
2134        /*
2135         * Look to see if there is anything else to do. Note, the completion
2136         * of the previous request may have caused a new request to be started
2137         * so be careful when doing this.
2138         */
2139
2140        if (!hs_ep->req && result >= 0)
2141                dwc2_gadget_start_next_request(hs_ep);
2142}
2143
2144/*
2145 * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA
2146 * @hs_ep: The endpoint the request was on.
2147 *
2148 * Get first request from the ep queue, determine descriptor on which complete
2149 * happened. SW discovers which descriptor currently in use by HW, adjusts
2150 * dma_address and calculates index of completed descriptor based on the value
2151 * of DEPDMA register. Update actual length of request, giveback to gadget.
2152 */
2153static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
2154{
2155        struct dwc2_hsotg *hsotg = hs_ep->parent;
2156        struct dwc2_hsotg_req *hs_req;
2157        struct usb_request *ureq;
2158        u32 desc_sts;
2159        u32 mask;
2160
2161        desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
2162
2163        /* Process only descriptors with buffer status set to DMA done */
2164        while ((desc_sts & DEV_DMA_BUFF_STS_MASK) >>
2165                DEV_DMA_BUFF_STS_SHIFT == DEV_DMA_BUFF_STS_DMADONE) {
2166
2167                hs_req = get_ep_head(hs_ep);
2168                if (!hs_req) {
2169                        dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__);
2170                        return;
2171                }
2172                ureq = &hs_req->req;
2173
2174                /* Check completion status */
2175                if ((desc_sts & DEV_DMA_STS_MASK) >> DEV_DMA_STS_SHIFT ==
2176                        DEV_DMA_STS_SUCC) {
2177                        mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK :
2178                                DEV_DMA_ISOC_RX_NBYTES_MASK;
2179                        ureq->actual = ureq->length - ((desc_sts & mask) >>
2180                                DEV_DMA_ISOC_NBYTES_SHIFT);
2181
2182                        /* Adjust actual len for ISOC Out if len is
2183                         * not align of 4
2184                         */
2185                        if (!hs_ep->dir_in && ureq->length & 0x3)
2186                                ureq->actual += 4 - (ureq->length & 0x3);
2187
2188                        /* Set actual frame number for completed transfers */
2189                        ureq->frame_number =
2190                                (desc_sts & DEV_DMA_ISOC_FRNUM_MASK) >>
2191                                DEV_DMA_ISOC_FRNUM_SHIFT;
2192                }
2193
2194                dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2195
2196                hs_ep->compl_desc++;
2197                if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_HS_ISOC - 1))
2198                        hs_ep->compl_desc = 0;
2199                desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
2200        }
2201}
2202
2203/*
2204 * dwc2_gadget_handle_isoc_bna - handle BNA interrupt for ISOC.
2205 * @hs_ep: The isochronous endpoint.
2206 *
2207 * If EP ISOC OUT then need to flush RX FIFO to remove source of BNA
2208 * interrupt. Reset target frame and next_desc to allow to start
2209 * ISOC's on NAK interrupt for IN direction or on OUTTKNEPDIS
2210 * interrupt for OUT direction.
2211 */
2212static void dwc2_gadget_handle_isoc_bna(struct dwc2_hsotg_ep *hs_ep)
2213{
2214        struct dwc2_hsotg *hsotg = hs_ep->parent;
2215
2216        if (!hs_ep->dir_in)
2217                dwc2_flush_rx_fifo(hsotg);
2218        dwc2_hsotg_complete_request(hsotg, hs_ep, get_ep_head(hs_ep), 0);
2219
2220        hs_ep->target_frame = TARGET_FRAME_INITIAL;
2221        hs_ep->next_desc = 0;
2222        hs_ep->compl_desc = 0;
2223}
2224
2225/**
2226 * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint
2227 * @hsotg: The device state.
2228 * @ep_idx: The endpoint index for the data
2229 * @size: The size of data in the fifo, in bytes
2230 *
2231 * The FIFO status shows there is data to read from the FIFO for a given
2232 * endpoint, so sort out whether we need to read the data into a request
2233 * that has been made for that endpoint.
2234 */
2235static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
2236{
2237        struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];
2238        struct dwc2_hsotg_req *hs_req = hs_ep->req;
2239        int to_read;
2240        int max_req;
2241        int read_ptr;
2242
2243        if (!hs_req) {
2244                u32 epctl = dwc2_readl(hsotg, DOEPCTL(ep_idx));
2245                int ptr;
2246
2247                dev_dbg(hsotg->dev,
2248                        "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n",
2249                         __func__, size, ep_idx, epctl);
2250
2251                /* dump the data from the FIFO, we've nothing we can do */
2252                for (ptr = 0; ptr < size; ptr += 4)
2253                        (void)dwc2_readl(hsotg, EPFIFO(ep_idx));
2254
2255                return;
2256        }
2257
2258        to_read = size;
2259        read_ptr = hs_req->req.actual;
2260        max_req = hs_req->req.length - read_ptr;
2261
2262        dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n",
2263                __func__, to_read, max_req, read_ptr, hs_req->req.length);
2264
2265        if (to_read > max_req) {
2266                /*
2267                 * more data appeared than we where willing
2268                 * to deal with in this request.
2269                 */
2270
2271                /* currently we don't deal this */
2272                WARN_ON_ONCE(1);
2273        }
2274
2275        hs_ep->total_data += to_read;
2276        hs_req->req.actual += to_read;
2277        to_read = DIV_ROUND_UP(to_read, 4);
2278
2279        /*
2280         * note, we might over-write the buffer end by 3 bytes depending on
2281         * alignment of the data.
2282         */
2283        dwc2_readl_rep(hsotg, EPFIFO(ep_idx),
2284                       hs_req->req.buf + read_ptr, to_read);
2285}
2286
2287/**
2288 * dwc2_hsotg_ep0_zlp - send/receive zero-length packet on control endpoint
2289 * @hsotg: The device instance
2290 * @dir_in: If IN zlp
2291 *
2292 * Generate a zero-length IN packet request for terminating a SETUP
2293 * transaction.
2294 *
2295 * Note, since we don't write any data to the TxFIFO, then it is
2296 * currently believed that we do not need to wait for any space in
2297 * the TxFIFO.
2298 */
2299static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in)
2300{
2301        /* eps_out[0] is used in both directions */
2302        hsotg->eps_out[0]->dir_in = dir_in;
2303        hsotg->ep0_state = dir_in ? DWC2_EP0_STATUS_IN : DWC2_EP0_STATUS_OUT;
2304
2305        dwc2_hsotg_program_zlp(hsotg, hsotg->eps_out[0]);
2306}
2307
2308static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg,
2309                                            u32 epctl_reg)
2310{
2311        u32 ctrl;
2312
2313        ctrl = dwc2_readl(hsotg, epctl_reg);
2314        if (ctrl & DXEPCTL_EOFRNUM)
2315                ctrl |= DXEPCTL_SETEVENFR;
2316        else
2317                ctrl |= DXEPCTL_SETODDFR;
2318        dwc2_writel(hsotg, ctrl, epctl_reg);
2319}
2320
2321/*
2322 * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc
2323 * @hs_ep - The endpoint on which transfer went
2324 *
2325 * Iterate over endpoints descriptor chain and get info on bytes remained
2326 * in DMA descriptors after transfer has completed. Used for non isoc EPs.
2327 */
2328static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
2329{
2330        const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
2331        struct dwc2_hsotg *hsotg = hs_ep->parent;
2332        unsigned int bytes_rem = 0;
2333        unsigned int bytes_rem_correction = 0;
2334        struct dwc2_dma_desc *desc = hs_ep->desc_list;
2335        int i;
2336        u32 status;
2337        u32 mps = hs_ep->ep.maxpacket;
2338        int dir_in = hs_ep->dir_in;
2339
2340        if (!desc)
2341                return -EINVAL;
2342
2343        /* Interrupt OUT EP with mps not multiple of 4 */
2344        if (hs_ep->index)
2345                if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4))
2346                        bytes_rem_correction = 4 - (mps % 4);
2347
2348        for (i = 0; i < hs_ep->desc_count; ++i) {
2349                status = desc->status;
2350                bytes_rem += status & DEV_DMA_NBYTES_MASK;
2351                bytes_rem -= bytes_rem_correction;
2352
2353                if (status & DEV_DMA_STS_MASK)
2354                        dev_err(hsotg->dev, "descriptor %d closed with %x\n",
2355                                i, status & DEV_DMA_STS_MASK);
2356
2357                if (status & DEV_DMA_L)
2358                        break;
2359
2360                desc++;
2361        }
2362
2363        return bytes_rem;
2364}
2365
2366/**
2367 * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO
2368 * @hsotg: The device instance
2369 * @epnum: The endpoint received from
2370 *
2371 * The RXFIFO has delivered an OutDone event, which means that the data
2372 * transfer for an OUT endpoint has been completed, either by a short
2373 * packet or by the finish of a transfer.
2374 */
2375static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
2376{
2377        u32 epsize = dwc2_readl(hsotg, DOEPTSIZ(epnum));
2378        struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum];
2379        struct dwc2_hsotg_req *hs_req = hs_ep->req;
2380        struct usb_request *req = &hs_req->req;
2381        unsigned int size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2382        int result = 0;
2383
2384        if (!hs_req) {
2385                dev_dbg(hsotg->dev, "%s: no request active\n", __func__);
2386                return;
2387        }
2388
2389        if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_OUT) {
2390                dev_dbg(hsotg->dev, "zlp packet received\n");
2391                dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2392                dwc2_hsotg_enqueue_setup(hsotg);
2393                return;
2394        }
2395
2396        if (using_desc_dma(hsotg))
2397                size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2398
2399        if (using_dma(hsotg)) {
2400                unsigned int size_done;
2401
2402                /*
2403                 * Calculate the size of the transfer by checking how much
2404                 * is left in the endpoint size register and then working it
2405                 * out from the amount we loaded for the transfer.
2406                 *
2407                 * We need to do this as DMA pointers are always 32bit aligned
2408                 * so may overshoot/undershoot the transfer.
2409                 */
2410
2411                size_done = hs_ep->size_loaded - size_left;
2412                size_done += hs_ep->last_load;
2413
2414                req->actual = size_done;
2415        }
2416
2417        /* if there is more request to do, schedule new transfer */
2418        if (req->actual < req->length && size_left == 0) {
2419                dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
2420                return;
2421        }
2422
2423        if (req->actual < req->length && req->short_not_ok) {
2424                dev_dbg(hsotg->dev, "%s: got %d/%d (short not ok) => error\n",
2425                        __func__, req->actual, req->length);
2426
2427                /*
2428                 * todo - what should we return here? there's no one else
2429                 * even bothering to check the status.
2430                 */
2431        }
2432
2433        /* DDMA IN status phase will start from StsPhseRcvd interrupt */
2434        if (!using_desc_dma(hsotg) && epnum == 0 &&
2435            hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
2436                /* Move to STATUS IN */
2437                if (!hsotg->delayed_status)
2438                        dwc2_hsotg_ep0_zlp(hsotg, true);
2439        }
2440
2441        /*
2442         * Slave mode OUT transfers do not go through XferComplete so
2443         * adjust the ISOC parity here.
2444         */
2445        if (!using_dma(hsotg)) {
2446                if (hs_ep->isochronous && hs_ep->interval == 1)
2447                        dwc2_hsotg_change_ep_iso_parity(hsotg, DOEPCTL(epnum));
2448                else if (hs_ep->isochronous && hs_ep->interval > 1)
2449                        dwc2_gadget_incr_frame_num(hs_ep);
2450        }
2451
2452        /* Set actual frame number for completed transfers */
2453        if (!using_desc_dma(hsotg) && hs_ep->isochronous)
2454                req->frame_number = hsotg->frame_number;
2455
2456        dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result);
2457}
2458
2459/**
2460 * dwc2_hsotg_handle_rx - RX FIFO has data
2461 * @hsotg: The device instance
2462 *
2463 * The IRQ handler has detected that the RX FIFO has some data in it
2464 * that requires processing, so find out what is in there and do the
2465 * appropriate read.
2466 *
2467 * The RXFIFO is a true FIFO, the packets coming out are still in packet
2468 * chunks, so if you have x packets received on an endpoint you'll get x
2469 * FIFO events delivered, each with a packet's worth of data in it.
2470 *
2471 * When using DMA, we should not be processing events from the RXFIFO
2472 * as the actual data should be sent to the memory directly and we turn
2473 * on the completion interrupts to get notifications of transfer completion.
2474 */
2475static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg)
2476{
2477        u32 grxstsr = dwc2_readl(hsotg, GRXSTSP);
2478        u32 epnum, status, size;
2479
2480        WARN_ON(using_dma(hsotg));
2481
2482        epnum = grxstsr & GRXSTS_EPNUM_MASK;
2483        status = grxstsr & GRXSTS_PKTSTS_MASK;
2484
2485        size = grxstsr & GRXSTS_BYTECNT_MASK;
2486        size >>= GRXSTS_BYTECNT_SHIFT;
2487
2488        dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n",
2489                __func__, grxstsr, size, epnum);
2490
2491        switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) {
2492        case GRXSTS_PKTSTS_GLOBALOUTNAK:
2493                dev_dbg(hsotg->dev, "GLOBALOUTNAK\n");
2494                break;
2495
2496        case GRXSTS_PKTSTS_OUTDONE:
2497                dev_dbg(hsotg->dev, "OutDone (Frame=0x%08x)\n",
2498                        dwc2_hsotg_read_frameno(hsotg));
2499
2500                if (!using_dma(hsotg))
2501                        dwc2_hsotg_handle_outdone(hsotg, epnum);
2502                break;
2503
2504        case GRXSTS_PKTSTS_SETUPDONE:
2505                dev_dbg(hsotg->dev,
2506                        "SetupDone (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
2507                        dwc2_hsotg_read_frameno(hsotg),
2508                        dwc2_readl(hsotg, DOEPCTL(0)));
2509                /*
2510                 * Call dwc2_hsotg_handle_outdone here if it was not called from
2511                 * GRXSTS_PKTSTS_OUTDONE. That is, if the core didn't
2512                 * generate GRXSTS_PKTSTS_OUTDONE for setup packet.
2513                 */
2514                if (hsotg->ep0_state == DWC2_EP0_SETUP)
2515                        dwc2_hsotg_handle_outdone(hsotg, epnum);
2516                break;
2517
2518        case GRXSTS_PKTSTS_OUTRX:
2519                dwc2_hsotg_rx_data(hsotg, epnum, size);
2520                break;
2521
2522        case GRXSTS_PKTSTS_SETUPRX:
2523                dev_dbg(hsotg->dev,
2524                        "SetupRX (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
2525                        dwc2_hsotg_read_frameno(hsotg),
2526                        dwc2_readl(hsotg, DOEPCTL(0)));
2527
2528                WARN_ON(hsotg->ep0_state != DWC2_EP0_SETUP);
2529
2530                dwc2_hsotg_rx_data(hsotg, epnum, size);
2531                break;
2532
2533        default:
2534                dev_warn(hsotg->dev, "%s: unknown status %08x\n",
2535                         __func__, grxstsr);
2536
2537                dwc2_hsotg_dump(hsotg);
2538                break;
2539        }
2540}
2541
2542/**
2543 * dwc2_hsotg_ep0_mps - turn max packet size into register setting
2544 * @mps: The maximum packet size in bytes.
2545 */
2546static u32 dwc2_hsotg_ep0_mps(unsigned int mps)
2547{
2548        switch (mps) {
2549        case 64:
2550                return D0EPCTL_MPS_64;
2551        case 32:
2552                return D0EPCTL_MPS_32;
2553        case 16:
2554                return D0EPCTL_MPS_16;
2555        case 8:
2556                return D0EPCTL_MPS_8;
2557        }
2558
2559        /* bad max packet size, warn and return invalid result */
2560        WARN_ON(1);
2561        return (u32)-1;
2562}
2563
2564/**
2565 * dwc2_hsotg_set_ep_maxpacket - set endpoint's max-packet field
2566 * @hsotg: The driver state.
2567 * @ep: The index number of the endpoint
2568 * @mps: The maximum packet size in bytes
2569 * @mc: The multicount value
2570 * @dir_in: True if direction is in.
2571 *
2572 * Configure the maximum packet size for the given endpoint, updating
2573 * the hardware control registers to reflect this.
2574 */
2575static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
2576                                        unsigned int ep, unsigned int mps,
2577                                        unsigned int mc, unsigned int dir_in)
2578{
2579        struct dwc2_hsotg_ep *hs_ep;
2580        u32 reg;
2581
2582        hs_ep = index_to_ep(hsotg, ep, dir_in);
2583        if (!hs_ep)
2584                return;
2585
2586        if (ep == 0) {
2587                u32 mps_bytes = mps;
2588
2589                /* EP0 is a special case */
2590                mps = dwc2_hsotg_ep0_mps(mps_bytes);
2591                if (mps > 3)
2592                        goto bad_mps;
2593                hs_ep->ep.maxpacket = mps_bytes;
2594                hs_ep->mc = 1;
2595        } else {
2596                if (mps > 1024)
2597                        goto bad_mps;
2598                hs_ep->mc = mc;
2599                if (mc > 3)
2600                        goto bad_mps;
2601                hs_ep->ep.maxpacket = mps;
2602        }
2603
2604        if (dir_in) {
2605                reg = dwc2_readl(hsotg, DIEPCTL(ep));
2606                reg &= ~DXEPCTL_MPS_MASK;
2607                reg |= mps;
2608                dwc2_writel(hsotg, reg, DIEPCTL(ep));
2609        } else {
2610                reg = dwc2_readl(hsotg, DOEPCTL(ep));
2611                reg &= ~DXEPCTL_MPS_MASK;
2612                reg |= mps;
2613                dwc2_writel(hsotg, reg, DOEPCTL(ep));
2614        }
2615
2616        return;
2617
2618bad_mps:
2619        dev_err(hsotg->dev, "ep%d: bad mps of %d\n", ep, mps);
2620}
2621
2622/**
2623 * dwc2_hsotg_txfifo_flush - flush Tx FIFO
2624 * @hsotg: The driver state
2625 * @idx: The index for the endpoint (0..15)
2626 */
2627static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx)
2628{
2629        dwc2_writel(hsotg, GRSTCTL_TXFNUM(idx) | GRSTCTL_TXFFLSH,
2630                    GRSTCTL);
2631
2632        /* wait until the fifo is flushed */
2633        if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL, GRSTCTL_TXFFLSH, 100))
2634                dev_warn(hsotg->dev, "%s: timeout flushing fifo GRSTCTL_TXFFLSH\n",
2635                         __func__);
2636}
2637
2638/**
2639 * dwc2_hsotg_trytx - check to see if anything needs transmitting
2640 * @hsotg: The driver state
2641 * @hs_ep: The driver endpoint to check.
2642 *
2643 * Check to see if there is a request that has data to send, and if so
2644 * make an attempt to write data into the FIFO.
2645 */
2646static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg,
2647                            struct dwc2_hsotg_ep *hs_ep)
2648{
2649        struct dwc2_hsotg_req *hs_req = hs_ep->req;
2650
2651        if (!hs_ep->dir_in || !hs_req) {
2652                /**
2653                 * if request is not enqueued, we disable interrupts
2654                 * for endpoints, excepting ep0
2655                 */
2656                if (hs_ep->index != 0)
2657                        dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index,
2658                                              hs_ep->dir_in, 0);
2659                return 0;
2660        }
2661
2662        if (hs_req->req.actual < hs_req->req.length) {
2663                dev_dbg(hsotg->dev, "trying to write more for ep%d\n",
2664                        hs_ep->index);
2665                return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
2666        }
2667
2668        return 0;
2669}
2670
2671/**
2672 * dwc2_hsotg_complete_in - complete IN transfer
2673 * @hsotg: The device state.
2674 * @hs_ep: The endpoint that has just completed.
2675 *
2676 * An IN transfer has been completed, update the transfer's state and then
2677 * call the relevant completion routines.
2678 */
2679static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
2680                                   struct dwc2_hsotg_ep *hs_ep)
2681{
2682        struct dwc2_hsotg_req *hs_req = hs_ep->req;
2683        u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index));
2684        int size_left, size_done;
2685
2686        if (!hs_req) {
2687                dev_dbg(hsotg->dev, "XferCompl but no req\n");
2688                return;
2689        }
2690
2691        /* Finish ZLP handling for IN EP0 transactions */
2692        if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {
2693                dev_dbg(hsotg->dev, "zlp packet sent\n");
2694
2695                /*
2696                 * While send zlp for DWC2_EP0_STATUS_IN EP direction was
2697                 * changed to IN. Change back to complete OUT transfer request
2698                 */
2699                hs_ep->dir_in = 0;
2700
2701                dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2702                if (hsotg->test_mode) {
2703                        int ret;
2704
2705                        ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode);
2706                        if (ret < 0) {
2707                                dev_dbg(hsotg->dev, "Invalid Test #%d\n",
2708                                        hsotg->test_mode);
2709                                dwc2_hsotg_stall_ep0(hsotg);
2710                                return;
2711                        }
2712                }
2713                dwc2_hsotg_enqueue_setup(hsotg);
2714                return;
2715        }
2716
2717        /*
2718         * Calculate the size of the transfer by checking how much is left
2719         * in the endpoint size register and then working it out from
2720         * the amount we loaded for the transfer.
2721         *
2722         * We do this even for DMA, as the transfer may have incremented
2723         * past the end of the buffer (DMA transfers are always 32bit
2724         * aligned).
2725         */
2726        if (using_desc_dma(hsotg)) {
2727                size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2728                if (size_left < 0)
2729                        dev_err(hsotg->dev, "error parsing DDMA results %d\n",
2730                                size_left);
2731        } else {
2732                size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2733        }
2734
2735        size_done = hs_ep->size_loaded - size_left;
2736        size_done += hs_ep->last_load;
2737
2738        if (hs_req->req.actual != size_done)
2739                dev_dbg(hsotg->dev, "%s: adjusting size done %d => %d\n",
2740                        __func__, hs_req->req.actual, size_done);
2741
2742        hs_req->req.actual = size_done;
2743        dev_dbg(hsotg->dev, "req->length:%d req->actual:%d req->zero:%d\n",
2744                hs_req->req.length, hs_req->req.actual, hs_req->req.zero);
2745
2746        if (!size_left && hs_req->req.actual < hs_req->req.length) {
2747                dev_dbg(hsotg->dev, "%s trying more for req...\n", __func__);
2748                dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
2749                return;
2750        }
2751
2752        /* Zlp for all endpoints in non DDMA, for ep0 only in DATA IN stage */
2753        if (hs_ep->send_zlp) {
2754                hs_ep->send_zlp = 0;
2755                if (!using_desc_dma(hsotg)) {
2756                        dwc2_hsotg_program_zlp(hsotg, hs_ep);
2757                        /* transfer will be completed on next complete interrupt */
2758                        return;
2759                }
2760        }
2761
2762        if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {
2763                /* Move to STATUS OUT */
2764                dwc2_hsotg_ep0_zlp(hsotg, false);
2765                return;
2766        }
2767
2768        dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2769}
2770
2771/**
2772 * dwc2_gadget_read_ep_interrupts - reads interrupts for given ep
2773 * @hsotg: The device state.
2774 * @idx: Index of ep.
2775 * @dir_in: Endpoint direction 1-in 0-out.
2776 *
2777 * Reads for endpoint with given index and direction, by masking
2778 * epint_reg with coresponding mask.
2779 */
2780static u32 dwc2_gadget_read_ep_interrupts(struct dwc2_hsotg *hsotg,
2781                                          unsigned int idx, int dir_in)
2782{
2783        u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
2784        u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
2785        u32 ints;
2786        u32 mask;
2787        u32 diepempmsk;
2788
2789        mask = dwc2_readl(hsotg, epmsk_reg);
2790        diepempmsk = dwc2_readl(hsotg, DIEPEMPMSK);
2791        mask |= ((diepempmsk >> idx) & 0x1) ? DIEPMSK_TXFIFOEMPTY : 0;
2792        mask |= DXEPINT_SETUP_RCVD;
2793
2794        ints = dwc2_readl(hsotg, epint_reg);
2795        ints &= mask;
2796        return ints;
2797}
2798
2799/**
2800 * dwc2_gadget_handle_ep_disabled - handle DXEPINT_EPDISBLD
2801 * @hs_ep: The endpoint on which interrupt is asserted.
2802 *
2803 * This interrupt indicates that the endpoint has been disabled per the
2804 * application's request.
2805 *
2806 * For IN endpoints flushes txfifo, in case of BULK clears DCTL_CGNPINNAK,
2807 * in case of ISOC completes current request.
2808 *
2809 * For ISOC-OUT endpoints completes expired requests. If there is remaining
2810 * request starts it.
2811 */
2812static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep)
2813{
2814        struct dwc2_hsotg *hsotg = hs_ep->parent;
2815        struct dwc2_hsotg_req *hs_req;
2816        unsigned char idx = hs_ep->index;
2817        int dir_in = hs_ep->dir_in;
2818        u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
2819        int dctl = dwc2_readl(hsotg, DCTL);
2820
2821        dev_dbg(hsotg->dev, "%s: EPDisbld\n", __func__);
2822
2823        if (dir_in) {
2824                int epctl = dwc2_readl(hsotg, epctl_reg);
2825
2826                dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
2827
2828                if (hs_ep->isochronous) {
2829                        dwc2_hsotg_complete_in(hsotg, hs_ep);
2830                        return;
2831                }
2832
2833                if ((epctl & DXEPCTL_STALL) && (epctl & DXEPCTL_EPTYPE_BULK)) {
2834                        int dctl = dwc2_readl(hsotg, DCTL);
2835
2836                        dctl |= DCTL_CGNPINNAK;
2837                        dwc2_writel(hsotg, dctl, DCTL);
2838                }
2839                return;
2840        }
2841
2842        if (dctl & DCTL_GOUTNAKSTS) {
2843                dctl |= DCTL_CGOUTNAK;
2844                dwc2_writel(hsotg, dctl, DCTL);
2845        }
2846
2847        if (!hs_ep->isochronous)
2848                return;
2849
2850        if (list_empty(&hs_ep->queue)) {
2851                dev_dbg(hsotg->dev, "%s: complete_ep 0x%p, ep->queue empty!\n",
2852                        __func__, hs_ep);
2853                return;
2854        }
2855
2856        do {
2857                hs_req = get_ep_head(hs_ep);
2858                if (hs_req)
2859                        dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req,
2860                                                    -ENODATA);
2861                dwc2_gadget_incr_frame_num(hs_ep);
2862                /* Update current frame number value. */
2863                hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
2864        } while (dwc2_gadget_target_frame_elapsed(hs_ep));
2865
2866        dwc2_gadget_start_next_request(hs_ep);
2867}
2868
2869/**
2870 * dwc2_gadget_handle_out_token_ep_disabled - handle DXEPINT_OUTTKNEPDIS
2871 * @ep: The endpoint on which interrupt is asserted.
2872 *
2873 * This is starting point for ISOC-OUT transfer, synchronization done with
2874 * first out token received from host while corresponding EP is disabled.
2875 *
2876 * Device does not know initial frame in which out token will come. For this
2877 * HW generates OUTTKNEPDIS - out token is received while EP is disabled. Upon
2878 * getting this interrupt SW starts calculation for next transfer frame.
2879 */
2880static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
2881{
2882        struct dwc2_hsotg *hsotg = ep->parent;
2883        int dir_in = ep->dir_in;
2884        u32 doepmsk;
2885
2886        if (dir_in || !ep->isochronous)
2887                return;
2888
2889        if (using_desc_dma(hsotg)) {
2890                if (ep->target_frame == TARGET_FRAME_INITIAL) {
2891                        /* Start first ISO Out */
2892                        ep->target_frame = hsotg->frame_number;
2893                        dwc2_gadget_start_isoc_ddma(ep);
2894                }
2895                return;
2896        }
2897
2898        if (ep->interval > 1 &&
2899            ep->target_frame == TARGET_FRAME_INITIAL) {
2900                u32 ctrl;
2901
2902                ep->target_frame = hsotg->frame_number;
2903                dwc2_gadget_incr_frame_num(ep);
2904
2905                ctrl = dwc2_readl(hsotg, DOEPCTL(ep->index));
2906                if (ep->target_frame & 0x1)
2907                        ctrl |= DXEPCTL_SETODDFR;
2908                else
2909                        ctrl |= DXEPCTL_SETEVENFR;
2910
2911                dwc2_writel(hsotg, ctrl, DOEPCTL(ep->index));
2912        }
2913
2914        dwc2_gadget_start_next_request(ep);
2915        doepmsk = dwc2_readl(hsotg, DOEPMSK);
2916        doepmsk &= ~DOEPMSK_OUTTKNEPDISMSK;
2917        dwc2_writel(hsotg, doepmsk, DOEPMSK);
2918}
2919
2920/**
2921 * dwc2_gadget_handle_nak - handle NAK interrupt
2922 * @hs_ep: The endpoint on which interrupt is asserted.
2923 *
2924 * This is starting point for ISOC-IN transfer, synchronization done with
2925 * first IN token received from host while corresponding EP is disabled.
2926 *
2927 * Device does not know when first one token will arrive from host. On first
2928 * token arrival HW generates 2 interrupts: 'in token received while FIFO empty'
2929 * and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was
2930 * sent in response to that as there was no data in FIFO. SW is basing on this
2931 * interrupt to obtain frame in which token has come and then based on the
2932 * interval calculates next frame for transfer.
2933 */
2934static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
2935{
2936        struct dwc2_hsotg *hsotg = hs_ep->parent;
2937        int dir_in = hs_ep->dir_in;
2938
2939        if (!dir_in || !hs_ep->isochronous)
2940                return;
2941
2942        if (hs_ep->target_frame == TARGET_FRAME_INITIAL) {
2943
2944                if (using_desc_dma(hsotg)) {
2945                        hs_ep->target_frame = hsotg->frame_number;
2946                        dwc2_gadget_incr_frame_num(hs_ep);
2947
2948                        /* In service interval mode target_frame must
2949                         * be set to last (u)frame of the service interval.
2950                         */
2951                        if (hsotg->params.service_interval) {
2952                                /* Set target_frame to the first (u)frame of
2953                                 * the service interval
2954                                 */
2955                                hs_ep->target_frame &= ~hs_ep->interval + 1;
2956
2957                                /* Set target_frame to the last (u)frame of
2958                                 * the service interval
2959                                 */
2960                                dwc2_gadget_incr_frame_num(hs_ep);
2961                                dwc2_gadget_dec_frame_num_by_one(hs_ep);
2962                        }
2963
2964                        dwc2_gadget_start_isoc_ddma(hs_ep);
2965                        return;
2966                }
2967
2968                hs_ep->target_frame = hsotg->frame_number;
2969                if (hs_ep->interval > 1) {
2970                        u32 ctrl = dwc2_readl(hsotg,
2971                                              DIEPCTL(hs_ep->index));
2972                        if (hs_ep->target_frame & 0x1)
2973                                ctrl |= DXEPCTL_SETODDFR;
2974                        else
2975                                ctrl |= DXEPCTL_SETEVENFR;
2976
2977                        dwc2_writel(hsotg, ctrl, DIEPCTL(hs_ep->index));
2978                }
2979
2980                dwc2_hsotg_complete_request(hsotg, hs_ep,
2981                                            get_ep_head(hs_ep), 0);
2982        }
2983
2984        if (!using_desc_dma(hsotg))
2985                dwc2_gadget_incr_frame_num(hs_ep);
2986}
2987
2988/**
2989 * dwc2_hsotg_epint - handle an in/out endpoint interrupt
2990 * @hsotg: The driver state
2991 * @idx: The index for the endpoint (0..15)
2992 * @dir_in: Set if this is an IN endpoint
2993 *
2994 * Process and clear any interrupt pending for an individual endpoint
2995 */
2996static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
2997                             int dir_in)
2998{
2999        struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);
3000        u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
3001        u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
3002        u32 epsiz_reg = dir_in ? DIEPTSIZ(idx) : DOEPTSIZ(idx);
3003        u32 ints;
3004
3005        ints = dwc2_gadget_read_ep_interrupts(hsotg, idx, dir_in);
3006
3007        /* Clear endpoint interrupts */
3008        dwc2_writel(hsotg, ints, epint_reg);
3009
3010        if (!hs_ep) {
3011                dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n",
3012                        __func__, idx, dir_in ? "in" : "out");
3013                return;
3014        }
3015
3016        dev_dbg(hsotg->dev, "%s: ep%d(%s) DxEPINT=0x%08x\n",
3017                __func__, idx, dir_in ? "in" : "out", ints);
3018
3019        /* Don't process XferCompl interrupt if it is a setup packet */
3020        if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD)))
3021                ints &= ~DXEPINT_XFERCOMPL;
3022
3023        /*
3024         * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP
3025         * stage and xfercomplete was generated without SETUP phase done
3026         * interrupt. SW should parse received setup packet only after host's
3027         * exit from setup phase of control transfer.
3028         */
3029        if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in &&
3030            hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP))
3031                ints &= ~DXEPINT_XFERCOMPL;
3032
3033        if (ints & DXEPINT_XFERCOMPL) {
3034                dev_dbg(hsotg->dev,
3035                        "%s: XferCompl: DxEPCTL=0x%08x, DXEPTSIZ=%08x\n",
3036                        __func__, dwc2_readl(hsotg, epctl_reg),
3037                        dwc2_readl(hsotg, epsiz_reg));
3038
3039                /* In DDMA handle isochronous requests separately */
3040                if (using_desc_dma(hsotg) && hs_ep->isochronous) {
3041                        /* XferCompl set along with BNA */
3042                        if (!(ints & DXEPINT_BNAINTR))
3043                                dwc2_gadget_complete_isoc_request_ddma(hs_ep);
3044                } else if (dir_in) {
3045                        /*
3046                         * We get OutDone from the FIFO, so we only
3047                         * need to look at completing IN requests here
3048                         * if operating slave mode
3049                         */
3050                        if (hs_ep->isochronous && hs_ep->interval > 1)
3051                                dwc2_gadget_incr_frame_num(hs_ep);
3052
3053                        dwc2_hsotg_complete_in(hsotg, hs_ep);
3054                        if (ints & DXEPINT_NAKINTRPT)
3055                                ints &= ~DXEPINT_NAKINTRPT;
3056
3057                        if (idx == 0 && !hs_ep->req)
3058                                dwc2_hsotg_enqueue_setup(hsotg);
3059                } else if (using_dma(hsotg)) {
3060                        /*
3061                         * We're using DMA, we need to fire an OutDone here
3062                         * as we ignore the RXFIFO.
3063                         */
3064                        if (hs_ep->isochronous && hs_ep->interval > 1)
3065                                dwc2_gadget_incr_frame_num(hs_ep);
3066
3067                        dwc2_hsotg_handle_outdone(hsotg, idx);
3068                }
3069        }
3070
3071        if (ints & DXEPINT_EPDISBLD)
3072                dwc2_gadget_handle_ep_disabled(hs_ep);
3073
3074        if (ints & DXEPINT_OUTTKNEPDIS)
3075                dwc2_gadget_handle_out_token_ep_disabled(hs_ep);
3076
3077        if (ints & DXEPINT_NAKINTRPT)
3078                dwc2_gadget_handle_nak(hs_ep);
3079
3080        if (ints & DXEPINT_AHBERR)
3081                dev_dbg(hsotg->dev, "%s: AHBErr\n", __func__);
3082
3083        if (ints & DXEPINT_SETUP) {  /* Setup or Timeout */
3084                dev_dbg(hsotg->dev, "%s: Setup/Timeout\n",  __func__);
3085
3086                if (using_dma(hsotg) && idx == 0) {
3087                        /*
3088                         * this is the notification we've received a
3089                         * setup packet. In non-DMA mode we'd get this
3090                         * from the RXFIFO, instead we need to process
3091                         * the setup here.
3092                         */
3093
3094                        if (dir_in)
3095                                WARN_ON_ONCE(1);
3096                        else
3097                                dwc2_hsotg_handle_outdone(hsotg, 0);
3098                }
3099        }
3100
3101        if (ints & DXEPINT_STSPHSERCVD) {
3102                dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__);
3103
3104                /* Safety check EP0 state when STSPHSERCVD asserted */
3105                if (hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
3106                        /* Move to STATUS IN for DDMA */
3107                        if (using_desc_dma(hsotg)) {
3108                                if (!hsotg->delayed_status)
3109                                        dwc2_hsotg_ep0_zlp(hsotg, true);
3110                                else
3111                                /* In case of 3 stage Control Write with delayed
3112                                 * status, when Status IN transfer started
3113                                 * before STSPHSERCVD asserted, NAKSTS bit not
3114                                 * cleared by CNAK in dwc2_hsotg_start_req()
3115                                 * function. Clear now NAKSTS to allow complete
3116                                 * transfer.
3117                                 */
3118                                        dwc2_set_bit(hsotg, DIEPCTL(0),
3119                                                     DXEPCTL_CNAK);
3120                        }
3121                }
3122
3123        }
3124
3125        if (ints & DXEPINT_BACK2BACKSETUP)
3126                dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__);
3127
3128        if (ints & DXEPINT_BNAINTR) {
3129                dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__);
3130                if (hs_ep->isochronous)
3131                        dwc2_gadget_handle_isoc_bna(hs_ep);
3132        }
3133
3134        if (dir_in && !hs_ep->isochronous) {
3135                /* not sure if this is important, but we'll clear it anyway */
3136                if (ints & DXEPINT_INTKNTXFEMP) {
3137                        dev_dbg(hsotg->dev, "%s: ep%d: INTknTXFEmpMsk\n",
3138                                __func__, idx);
3139                }
3140
3141                /* this probably means something bad is happening */
3142                if (ints & DXEPINT_INTKNEPMIS) {
3143                        dev_warn(hsotg->dev, "%s: ep%d: INTknEP\n",
3144                                 __func__, idx);
3145                }
3146
3147                /* FIFO has space or is empty (see GAHBCFG) */
3148                if (hsotg->dedicated_fifos &&
3149                    ints & DXEPINT_TXFEMP) {
3150                        dev_dbg(hsotg->dev, "%s: ep%d: TxFIFOEmpty\n",
3151                                __func__, idx);
3152                        if (!using_dma(hsotg))
3153                                dwc2_hsotg_trytx(hsotg, hs_ep);
3154                }
3155        }
3156}
3157
3158/**
3159 * dwc2_hsotg_irq_enumdone - Handle EnumDone interrupt (enumeration done)
3160 * @hsotg: The device state.
3161 *
3162 * Handle updating the device settings after the enumeration phase has
3163 * been completed.
3164 */
3165static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
3166{
3167        u32 dsts = dwc2_readl(hsotg, DSTS);
3168        int ep0_mps = 0, ep_mps = 8;
3169
3170        /*
3171         * This should signal the finish of the enumeration phase
3172         * of the USB handshaking, so we should now know what rate
3173         * we connected at.
3174         */
3175
3176        dev_dbg(hsotg->dev, "EnumDone (DSTS=0x%08x)\n", dsts);
3177
3178        /*
3179         * note, since we're limited by the size of transfer on EP0, and
3180         * it seems IN transfers must be a even number of packets we do
3181         * not advertise a 64byte MPS on EP0.
3182         */
3183
3184        /* catch both EnumSpd_FS and EnumSpd_FS48 */
3185        switch ((dsts & DSTS_ENUMSPD_MASK) >> DSTS_ENUMSPD_SHIFT) {
3186        case DSTS_ENUMSPD_FS:
3187        case DSTS_ENUMSPD_FS48:
3188                hsotg->gadget.speed = USB_SPEED_FULL;
3189                ep0_mps = EP0_MPS_LIMIT;
3190                ep_mps = 1023;
3191                break;
3192
3193        case DSTS_ENUMSPD_HS:
3194                hsotg->gadget.speed = USB_SPEED_HIGH;
3195                ep0_mps = EP0_MPS_LIMIT;
3196                ep_mps = 1024;
3197                break;
3198
3199        case DSTS_ENUMSPD_LS:
3200                hsotg->gadget.speed = USB_SPEED_LOW;
3201                ep0_mps = 8;
3202                ep_mps = 8;
3203                /*
3204                 * note, we don't actually support LS in this driver at the
3205                 * moment, and the documentation seems to imply that it isn't
3206                 * supported by the PHYs on some of the devices.
3207                 */
3208                break;
3209        }
3210        dev_info(hsotg->dev, "new device is %s\n",
3211                 usb_speed_string(hsotg->gadget.speed));
3212
3213        /*
3214         * we should now know the maximum packet size for an
3215         * endpoint, so set the endpoints to a default value.
3216         */
3217
3218        if (ep0_mps) {
3219                int i;
3220                /* Initialize ep0 for both in and out directions */
3221                dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1);
3222                dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0);
3223                for (i = 1; i < hsotg->num_of_eps; i++) {
3224                        if (hsotg->eps_in[i])
3225                                dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3226                                                            0, 1);
3227                        if (hsotg->eps_out[i])
3228                                dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3229                                                            0, 0);
3230                }
3231        }
3232
3233        /* ensure after enumeration our EP0 is active */
3234
3235        dwc2_hsotg_enqueue_setup(hsotg);
3236
3237        dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3238                dwc2_readl(hsotg, DIEPCTL0),
3239                dwc2_readl(hsotg, DOEPCTL0));
3240}
3241
3242/**
3243 * kill_all_requests - remove all requests from the endpoint's queue
3244 * @hsotg: The device state.
3245 * @ep: The endpoint the requests may be on.
3246 * @result: The result code to use.
3247 *
3248 * Go through the requests on the given endpoint and mark them
3249 * completed with the given result code.
3250 */
3251static void kill_all_requests(struct dwc2_hsotg *hsotg,
3252                              struct dwc2_hsotg_ep *ep,
3253                              int result)
3254{
3255        unsigned int size;
3256
3257        ep->req = NULL;
3258
3259        while (!list_empty(&ep->queue)) {
3260                struct dwc2_hsotg_req *req = get_ep_head(ep);
3261
3262                dwc2_hsotg_complete_request(hsotg, ep, req, result);
3263        }
3264
3265        if (!hsotg->dedicated_fifos)
3266                return;
3267        size = (dwc2_readl(hsotg, DTXFSTS(ep->fifo_index)) & 0xffff) * 4;
3268        if (size < ep->fifo_size)
3269                dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index);
3270}
3271
3272/**
3273 * dwc2_hsotg_disconnect - disconnect service
3274 * @hsotg: The device state.
3275 *
3276 * The device has been disconnected. Remove all current
3277 * transactions and signal the gadget driver that this
3278 * has happened.
3279 */
3280void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg)
3281{
3282        unsigned int ep;
3283
3284        if (!hsotg->connected)
3285                return;
3286
3287        hsotg->connected = 0;
3288        hsotg->test_mode = 0;
3289
3290        /* all endpoints should be shutdown */
3291        for (ep = 0; ep < hsotg->num_of_eps; ep++) {
3292                if (hsotg->eps_in[ep])
3293                        kill_all_requests(hsotg, hsotg->eps_in[ep],
3294                                          -ESHUTDOWN);
3295                if (hsotg->eps_out[ep])
3296                        kill_all_requests(hsotg, hsotg->eps_out[ep],
3297                                          -ESHUTDOWN);
3298        }
3299
3300        call_gadget(hsotg, disconnect);
3301        hsotg->lx_state = DWC2_L3;
3302
3303        usb_gadget_set_state(&hsotg->gadget, USB_STATE_NOTATTACHED);
3304}
3305
3306/**
3307 * dwc2_hsotg_irq_fifoempty - TX FIFO empty interrupt handler
3308 * @hsotg: The device state:
3309 * @periodic: True if this is a periodic FIFO interrupt
3310 */
3311static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic)
3312{
3313        struct dwc2_hsotg_ep *ep;
3314        int epno, ret;
3315
3316        /* look through for any more data to transmit */
3317        for (epno = 0; epno < hsotg->num_of_eps; epno++) {
3318                ep = index_to_ep(hsotg, epno, 1);
3319
3320                if (!ep)
3321                        continue;
3322
3323                if (!ep->dir_in)
3324                        continue;
3325
3326                if ((periodic && !ep->periodic) ||
3327                    (!periodic && ep->periodic))
3328                        continue;
3329
3330                ret = dwc2_hsotg_trytx(hsotg, ep);
3331                if (ret < 0)
3332                        break;
3333        }
3334}
3335
3336/* IRQ flags which will trigger a retry around the IRQ loop */
3337#define IRQ_RETRY_MASK (GINTSTS_NPTXFEMP | \
3338                        GINTSTS_PTXFEMP |  \
3339                        GINTSTS_RXFLVL)
3340
3341static int dwc2_hsotg_ep_disable(struct usb_ep *ep);
3342/**
3343 * dwc2_hsotg_core_init_disconnected - issue softreset to the core
3344 * @hsotg: The device state
3345 * @is_usb_reset: Usb resetting flag
3346 *
3347 * Issue a soft reset to the core, and await the core finishing it.
3348 */
3349void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
3350                                       bool is_usb_reset)
3351{
3352        u32 intmsk;
3353        u32 val;
3354        u32 usbcfg;
3355        u32 dcfg = 0;
3356        int ep;
3357
3358        /* Kill any ep0 requests as controller will be reinitialized */
3359        kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET);
3360
3361        if (!is_usb_reset) {
3362                if (dwc2_core_reset(hsotg, true))
3363                        return;
3364        } else {
3365                /* all endpoints should be shutdown */
3366                for (ep = 1; ep < hsotg->num_of_eps; ep++) {
3367                        if (hsotg->eps_in[ep])
3368                                dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
3369                        if (hsotg->eps_out[ep])
3370                                dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
3371                }
3372        }
3373
3374        /*
3375         * we must now enable ep0 ready for host detection and then
3376         * set configuration.
3377         */
3378
3379        /* keep other bits untouched (so e.g. forced modes are not lost) */
3380        usbcfg = dwc2_readl(hsotg, GUSBCFG);
3381        usbcfg &= ~GUSBCFG_TOUTCAL_MASK;
3382        usbcfg |= GUSBCFG_TOUTCAL(7);
3383
3384        /* remove the HNP/SRP and set the PHY */
3385        usbcfg &= ~(GUSBCFG_SRPCAP | GUSBCFG_HNPCAP);
3386        dwc2_writel(hsotg, usbcfg, GUSBCFG);
3387
3388        dwc2_phy_init(hsotg, true);
3389
3390        dwc2_hsotg_init_fifo(hsotg);
3391
3392        if (!is_usb_reset)
3393                dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
3394
3395        dcfg |= DCFG_EPMISCNT(1);
3396
3397        switch (hsotg->params.speed) {
3398        case DWC2_SPEED_PARAM_LOW:
3399                dcfg |= DCFG_DEVSPD_LS;
3400                break;
3401        case DWC2_SPEED_PARAM_FULL:
3402                if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS)
3403                        dcfg |= DCFG_DEVSPD_FS48;
3404                else
3405                        dcfg |= DCFG_DEVSPD_FS;
3406                break;
3407        default:
3408                dcfg |= DCFG_DEVSPD_HS;
3409        }
3410
3411        if (hsotg->params.ipg_isoc_en)
3412                dcfg |= DCFG_IPG_ISOC_SUPPORDED;
3413
3414        dwc2_writel(hsotg, dcfg,  DCFG);
3415
3416        /* Clear any pending OTG interrupts */
3417        dwc2_writel(hsotg, 0xffffffff, GOTGINT);
3418
3419        /* Clear any pending interrupts */
3420        dwc2_writel(hsotg, 0xffffffff, GINTSTS);
3421        intmsk = GINTSTS_ERLYSUSP | GINTSTS_SESSREQINT |
3422                GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF |
3423                GINTSTS_USBRST | GINTSTS_RESETDET |
3424                GINTSTS_ENUMDONE | GINTSTS_OTGINT |
3425                GINTSTS_USBSUSP | GINTSTS_WKUPINT |
3426                GINTSTS_LPMTRANRCVD;
3427
3428        if (!using_desc_dma(hsotg))
3429                intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
3430
3431        if (!hsotg->params.external_id_pin_ctl)
3432                intmsk |= GINTSTS_CONIDSTSCHNG;
3433
3434        dwc2_writel(hsotg, intmsk, GINTMSK);
3435
3436        if (using_dma(hsotg)) {
3437                dwc2_writel(hsotg, GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN |
3438                            hsotg->params.ahbcfg,
3439                            GAHBCFG);
3440
3441                /* Set DDMA mode support in the core if needed */
3442                if (using_desc_dma(hsotg))
3443                        dwc2_set_bit(hsotg, DCFG, DCFG_DESCDMA_EN);
3444
3445        } else {
3446                dwc2_writel(hsotg, ((hsotg->dedicated_fifos) ?
3447                                                (GAHBCFG_NP_TXF_EMP_LVL |
3448                                                 GAHBCFG_P_TXF_EMP_LVL) : 0) |
3449                            GAHBCFG_GLBL_INTR_EN, GAHBCFG);
3450        }
3451
3452        /*
3453         * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts
3454         * when we have no data to transfer. Otherwise we get being flooded by
3455         * interrupts.
3456         */
3457
3458        dwc2_writel(hsotg, ((hsotg->dedicated_fifos && !using_dma(hsotg)) ?
3459                DIEPMSK_TXFIFOEMPTY | DIEPMSK_INTKNTXFEMPMSK : 0) |
3460                DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK |
3461                DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK,
3462                DIEPMSK);
3463
3464        /*
3465         * don't need XferCompl, we get that from RXFIFO in slave mode. In
3466         * DMA mode we may need this and StsPhseRcvd.
3467         */
3468        dwc2_writel(hsotg, (using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK |
3469                DOEPMSK_STSPHSERCVDMSK) : 0) |
3470                DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK |
3471                DOEPMSK_SETUPMSK,
3472                DOEPMSK);
3473
3474        /* Enable BNA interrupt for DDMA */
3475        if (using_desc_dma(hsotg)) {
3476                dwc2_set_bit(hsotg, DOEPMSK, DOEPMSK_BNAMSK);
3477                dwc2_set_bit(hsotg, DIEPMSK, DIEPMSK_BNAININTRMSK);
3478        }
3479
3480        /* Enable Service Interval mode if supported */
3481        if (using_desc_dma(hsotg) && hsotg->params.service_interval)
3482                dwc2_set_bit(hsotg, DCTL, DCTL_SERVICE_INTERVAL_SUPPORTED);
3483
3484        dwc2_writel(hsotg, 0, DAINTMSK);
3485
3486        dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3487                dwc2_readl(hsotg, DIEPCTL0),
3488                dwc2_readl(hsotg, DOEPCTL0));
3489
3490        /* enable in and out endpoint interrupts */
3491        dwc2_hsotg_en_gsint(hsotg, GINTSTS_OEPINT | GINTSTS_IEPINT);
3492
3493        /*
3494         * Enable the RXFIFO when in slave mode, as this is how we collect
3495         * the data. In DMA mode, we get events from the FIFO but also
3496         * things we cannot process, so do not use it.
3497         */
3498        if (!using_dma(hsotg))
3499                dwc2_hsotg_en_gsint(hsotg, GINTSTS_RXFLVL);
3500
3501        /* Enable interrupts for EP0 in and out */
3502        dwc2_hsotg_ctrl_epint(hsotg, 0, 0, 1);
3503        dwc2_hsotg_ctrl_epint(hsotg, 0, 1, 1);
3504
3505        if (!is_usb_reset) {
3506                dwc2_set_bit(hsotg, DCTL, DCTL_PWRONPRGDONE);
3507                udelay(10);  /* see openiboot */
3508                dwc2_clear_bit(hsotg, DCTL, DCTL_PWRONPRGDONE);
3509        }
3510
3511        dev_dbg(hsotg->dev, "DCTL=0x%08x\n", dwc2_readl(hsotg, DCTL));
3512
3513        /*
3514         * DxEPCTL_USBActEp says RO in manual, but seems to be set by
3515         * writing to the EPCTL register..
3516         */
3517
3518        /* set to read 1 8byte packet */
3519        dwc2_writel(hsotg, DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
3520               DXEPTSIZ_XFERSIZE(8), DOEPTSIZ0);
3521
3522        dwc2_writel(hsotg, dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
3523               DXEPCTL_CNAK | DXEPCTL_EPENA |
3524               DXEPCTL_USBACTEP,
3525               DOEPCTL0);
3526
3527        /* enable, but don't activate EP0in */
3528        dwc2_writel(hsotg, dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
3529               DXEPCTL_USBACTEP, DIEPCTL0);
3530
3531        /* clear global NAKs */
3532        val = DCTL_CGOUTNAK | DCTL_CGNPINNAK;
3533        if (!is_usb_reset)
3534                val |= DCTL_SFTDISCON;
3535        dwc2_set_bit(hsotg, DCTL, val);
3536
3537        /* configure the core to support LPM */
3538        dwc2_gadget_init_lpm(hsotg);
3539
3540        /* program GREFCLK register if needed */
3541        if (using_desc_dma(hsotg) && hsotg->params.service_interval)
3542                dwc2_gadget_program_ref_clk(hsotg);
3543
3544        /* must be at-least 3ms to allow bus to see disconnect */
3545        mdelay(3);
3546
3547        hsotg->lx_state = DWC2_L0;
3548
3549        dwc2_hsotg_enqueue_setup(hsotg);
3550
3551        dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3552                dwc2_readl(hsotg, DIEPCTL0),
3553                dwc2_readl(hsotg, DOEPCTL0));
3554}
3555
3556void dwc2_hsotg_core_disconnect(struct dwc2_hsotg *hsotg)
3557{
3558        /* set the soft-disconnect bit */
3559        dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
3560}
3561
3562void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg)
3563{
3564        /* remove the soft-disconnect and let's go */
3565        dwc2_clear_bit(hsotg, DCTL, DCTL_SFTDISCON);
3566}
3567
3568/**
3569 * dwc2_gadget_handle_incomplete_isoc_in - handle incomplete ISO IN Interrupt.
3570 * @hsotg: The device state:
3571 *
3572 * This interrupt indicates one of the following conditions occurred while
3573 * transmitting an ISOC transaction.
3574 * - Corrupted IN Token for ISOC EP.
3575 * - Packet not complete in FIFO.
3576 *
3577 * The following actions will be taken:
3578 * - Determine the EP
3579 * - Disable EP; when 'Endpoint Disabled' interrupt is received Flush FIFO
3580 */
3581static void dwc2_gadget_handle_incomplete_isoc_in(struct dwc2_hsotg *hsotg)
3582{
3583        struct dwc2_hsotg_ep *hs_ep;
3584        u32 epctrl;
3585        u32 daintmsk;
3586        u32 idx;
3587
3588        dev_dbg(hsotg->dev, "Incomplete isoc in interrupt received:\n");
3589
3590        daintmsk = dwc2_readl(hsotg, DAINTMSK);
3591
3592        for (idx = 1; idx < hsotg->num_of_eps; idx++) {
3593                hs_ep = hsotg->eps_in[idx];
3594                /* Proceed only unmasked ISOC EPs */
3595                if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous)
3596                        continue;
3597
3598                epctrl = dwc2_readl(hsotg, DIEPCTL(idx));
3599                if ((epctrl & DXEPCTL_EPENA) &&
3600                    dwc2_gadget_target_frame_elapsed(hs_ep)) {
3601                        epctrl |= DXEPCTL_SNAK;
3602                        epctrl |= DXEPCTL_EPDIS;
3603                        dwc2_writel(hsotg, epctrl, DIEPCTL(idx));
3604                }
3605        }
3606
3607        /* Clear interrupt */
3608        dwc2_writel(hsotg, GINTSTS_INCOMPL_SOIN, GINTSTS);
3609}
3610
3611/**
3612 * dwc2_gadget_handle_incomplete_isoc_out - handle incomplete ISO OUT Interrupt
3613 * @hsotg: The device state:
3614 *
3615 * This interrupt indicates one of the following conditions occurred while
3616 * transmitting an ISOC transaction.
3617 * - Corrupted OUT Token for ISOC EP.
3618 * - Packet not complete in FIFO.
3619 *
3620 * The following actions will be taken:
3621 * - Determine the EP
3622 * - Set DCTL_SGOUTNAK and unmask GOUTNAKEFF if target frame elapsed.
3623 */
3624static void dwc2_gadget_handle_incomplete_isoc_out(struct dwc2_hsotg *hsotg)
3625{
3626        u32 gintsts;
3627        u32 gintmsk;
3628        u32 daintmsk;
3629        u32 epctrl;
3630        struct dwc2_hsotg_ep *hs_ep;
3631        int idx;
3632
3633        dev_dbg(hsotg->dev, "%s: GINTSTS_INCOMPL_SOOUT\n", __func__);
3634
3635        daintmsk = dwc2_readl(hsotg, DAINTMSK);
3636        daintmsk >>= DAINT_OUTEP_SHIFT;
3637
3638        for (idx = 1; idx < hsotg->num_of_eps; idx++) {
3639                hs_ep = hsotg->eps_out[idx];
3640                /* Proceed only unmasked ISOC EPs */
3641                if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous)
3642                        continue;
3643
3644                epctrl = dwc2_readl(hsotg, DOEPCTL(idx));
3645                if ((epctrl & DXEPCTL_EPENA) &&
3646                    dwc2_gadget_target_frame_elapsed(hs_ep)) {
3647                        /* Unmask GOUTNAKEFF interrupt */
3648                        gintmsk = dwc2_readl(hsotg, GINTMSK);
3649                        gintmsk |= GINTSTS_GOUTNAKEFF;
3650                        dwc2_writel(hsotg, gintmsk, GINTMSK);
3651
3652                        gintsts = dwc2_readl(hsotg, GINTSTS);
3653                        if (!(gintsts & GINTSTS_GOUTNAKEFF)) {
3654                                dwc2_set_bit(hsotg, DCTL, DCTL_SGOUTNAK);
3655                                break;
3656                        }
3657                }
3658        }
3659
3660        /* Clear interrupt */
3661        dwc2_writel(hsotg, GINTSTS_INCOMPL_SOOUT, GINTSTS);
3662}
3663
3664/**
3665 * dwc2_hsotg_irq - handle device interrupt
3666 * @irq: The IRQ number triggered
3667 * @pw: The pw value when registered the handler.
3668 */
3669static irqreturn_t dwc2_hsotg_irq(int irq, void *pw)
3670{
3671        struct dwc2_hsotg *hsotg = pw;
3672        int retry_count = 8;
3673        u32 gintsts;
3674        u32 gintmsk;
3675
3676        if (!dwc2_is_device_mode(hsotg))
3677                return IRQ_NONE;
3678
3679        spin_lock(&hsotg->lock);
3680irq_retry:
3681        gintsts = dwc2_readl(hsotg, GINTSTS);
3682        gintmsk = dwc2_readl(hsotg, GINTMSK);
3683
3684        dev_dbg(hsotg->dev, "%s: %08x %08x (%08x) retry %d\n",
3685                __func__, gintsts, gintsts & gintmsk, gintmsk, retry_count);
3686
3687        gintsts &= gintmsk;
3688
3689        if (gintsts & GINTSTS_RESETDET) {
3690                dev_dbg(hsotg->dev, "%s: USBRstDet\n", __func__);
3691
3692                dwc2_writel(hsotg, GINTSTS_RESETDET, GINTSTS);
3693
3694                /* This event must be used only if controller is suspended */
3695                if (hsotg->in_ppd && hsotg->lx_state == DWC2_L2)
3696                        dwc2_exit_partial_power_down(hsotg, 0, true);
3697
3698                hsotg->lx_state = DWC2_L0;
3699        }
3700
3701        if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) {
3702                u32 usb_status = dwc2_readl(hsotg, GOTGCTL);
3703                u32 connected = hsotg->connected;
3704
3705                dev_dbg(hsotg->dev, "%s: USBRst\n", __func__);
3706                dev_dbg(hsotg->dev, "GNPTXSTS=%08x\n",
3707                        dwc2_readl(hsotg, GNPTXSTS));
3708
3709                dwc2_writel(hsotg, GINTSTS_USBRST, GINTSTS);
3710
3711                /* Report disconnection if it is not already done. */
3712                dwc2_hsotg_disconnect(hsotg);
3713
3714                /* Reset device address to zero */
3715                dwc2_clear_bit(hsotg, DCFG, DCFG_DEVADDR_MASK);
3716
3717                if (usb_status & GOTGCTL_BSESVLD && connected)
3718                        dwc2_hsotg_core_init_disconnected(hsotg, true);
3719        }
3720
3721        if (gintsts & GINTSTS_ENUMDONE) {
3722                dwc2_writel(hsotg, GINTSTS_ENUMDONE, GINTSTS);
3723
3724                dwc2_hsotg_irq_enumdone(hsotg);
3725        }
3726
3727        if (gintsts & (GINTSTS_OEPINT | GINTSTS_IEPINT)) {
3728                u32 daint = dwc2_readl(hsotg, DAINT);
3729                u32 daintmsk = dwc2_readl(hsotg, DAINTMSK);
3730                u32 daint_out, daint_in;
3731                int ep;
3732
3733                daint &= daintmsk;
3734                daint_out = daint >> DAINT_OUTEP_SHIFT;
3735                daint_in = daint & ~(daint_out << DAINT_OUTEP_SHIFT);
3736
3737                dev_dbg(hsotg->dev, "%s: daint=%08x\n", __func__, daint);
3738
3739                for (ep = 0; ep < hsotg->num_of_eps && daint_out;
3740                                                ep++, daint_out >>= 1) {
3741                        if (daint_out & 1)
3742                                dwc2_hsotg_epint(hsotg, ep, 0);
3743                }
3744
3745                for (ep = 0; ep < hsotg->num_of_eps  && daint_in;
3746                                                ep++, daint_in >>= 1) {
3747                        if (daint_in & 1)
3748                                dwc2_hsotg_epint(hsotg, ep, 1);
3749                }
3750        }
3751
3752        /* check both FIFOs */
3753
3754        if (gintsts & GINTSTS_NPTXFEMP) {
3755                dev_dbg(hsotg->dev, "NPTxFEmp\n");
3756
3757                /*
3758                 * Disable the interrupt to stop it happening again
3759                 * unless one of these endpoint routines decides that
3760                 * it needs re-enabling
3761                 */
3762
3763                dwc2_hsotg_disable_gsint(hsotg, GINTSTS_NPTXFEMP);
3764                dwc2_hsotg_irq_fifoempty(hsotg, false);
3765        }
3766
3767        if (gintsts & GINTSTS_PTXFEMP) {
3768                dev_dbg(hsotg->dev, "PTxFEmp\n");
3769
3770                /* See note in GINTSTS_NPTxFEmp */
3771
3772                dwc2_hsotg_disable_gsint(hsotg, GINTSTS_PTXFEMP);
3773                dwc2_hsotg_irq_fifoempty(hsotg, true);
3774        }
3775
3776        if (gintsts & GINTSTS_RXFLVL) {
3777                /*
3778                 * note, since GINTSTS_RxFLvl doubles as FIFO-not-empty,
3779                 * we need to retry dwc2_hsotg_handle_rx if this is still
3780                 * set.
3781                 */
3782
3783                dwc2_hsotg_handle_rx(hsotg);
3784        }
3785
3786        if (gintsts & GINTSTS_ERLYSUSP) {
3787                dev_dbg(hsotg->dev, "GINTSTS_ErlySusp\n");
3788                dwc2_writel(hsotg, GINTSTS_ERLYSUSP, GINTSTS);
3789        }
3790
3791        /*
3792         * these next two seem to crop-up occasionally causing the core
3793         * to shutdown the USB transfer, so try clearing them and logging
3794         * the occurrence.
3795         */
3796
3797        if (gintsts & GINTSTS_GOUTNAKEFF) {
3798                u8 idx;
3799                u32 epctrl;
3800                u32 gintmsk;
3801                u32 daintmsk;
3802                struct dwc2_hsotg_ep *hs_ep;
3803
3804                daintmsk = dwc2_readl(hsotg, DAINTMSK);
3805                daintmsk >>= DAINT_OUTEP_SHIFT;
3806                /* Mask this interrupt */
3807                gintmsk = dwc2_readl(hsotg, GINTMSK);
3808                gintmsk &= ~GINTSTS_GOUTNAKEFF;
3809                dwc2_writel(hsotg, gintmsk, GINTMSK);
3810
3811                dev_dbg(hsotg->dev, "GOUTNakEff triggered\n");
3812                for (idx = 1; idx < hsotg->num_of_eps; idx++) {
3813                        hs_ep = hsotg->eps_out[idx];
3814                        /* Proceed only unmasked ISOC EPs */
3815                        if (BIT(idx) & ~daintmsk)
3816                                continue;
3817
3818                        epctrl = dwc2_readl(hsotg, DOEPCTL(idx));
3819
3820                        //ISOC Ep's only
3821                        if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) {
3822                                epctrl |= DXEPCTL_SNAK;
3823                                epctrl |= DXEPCTL_EPDIS;
3824                                dwc2_writel(hsotg, epctrl, DOEPCTL(idx));
3825                                continue;
3826                        }
3827
3828                        //Non-ISOC EP's
3829                        if (hs_ep->halted) {
3830                                if (!(epctrl & DXEPCTL_EPENA))
3831                                        epctrl |= DXEPCTL_EPENA;
3832                                epctrl |= DXEPCTL_EPDIS;
3833                                epctrl |= DXEPCTL_STALL;
3834                                dwc2_writel(hsotg, epctrl, DOEPCTL(idx));
3835                        }
3836                }
3837
3838                /* This interrupt bit is cleared in DXEPINT_EPDISBLD handler */
3839        }
3840
3841        if (gintsts & GINTSTS_GINNAKEFF) {
3842                dev_info(hsotg->dev, "GINNakEff triggered\n");
3843
3844                dwc2_set_bit(hsotg, DCTL, DCTL_CGNPINNAK);
3845
3846                dwc2_hsotg_dump(hsotg);
3847        }
3848
3849        if (gintsts & GINTSTS_INCOMPL_SOIN)
3850                dwc2_gadget_handle_incomplete_isoc_in(hsotg);
3851
3852        if (gintsts & GINTSTS_INCOMPL_SOOUT)
3853                dwc2_gadget_handle_incomplete_isoc_out(hsotg);
3854
3855        /*
3856         * if we've had fifo events, we should try and go around the
3857         * loop again to see if there's any point in returning yet.
3858         */
3859
3860        if (gintsts & IRQ_RETRY_MASK && --retry_count > 0)
3861                goto irq_retry;
3862
3863        /* Check WKUP_ALERT interrupt*/
3864        if (hsotg->params.service_interval)
3865                dwc2_gadget_wkup_alert_handler(hsotg);
3866
3867        spin_unlock(&hsotg->lock);
3868
3869        return IRQ_HANDLED;
3870}
3871
3872static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
3873                                   struct dwc2_hsotg_ep *hs_ep)
3874{
3875        u32 epctrl_reg;
3876        u32 epint_reg;
3877
3878        epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
3879                DOEPCTL(hs_ep->index);
3880        epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
3881                DOEPINT(hs_ep->index);
3882
3883        dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
3884                hs_ep->name);
3885
3886        if (hs_ep->dir_in) {
3887                if (hsotg->dedicated_fifos || hs_ep->periodic) {
3888                        dwc2_set_bit(hsotg, epctrl_reg, DXEPCTL_SNAK);
3889                        /* Wait for Nak effect */
3890                        if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
3891                                                    DXEPINT_INEPNAKEFF, 100))
3892                                dev_warn(hsotg->dev,
3893                                         "%s: timeout DIEPINT.NAKEFF\n",
3894                                         __func__);
3895                } else {
3896                        dwc2_set_bit(hsotg, DCTL, DCTL_SGNPINNAK);
3897                        /* Wait for Nak effect */
3898                        if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3899                                                    GINTSTS_GINNAKEFF, 100))
3900                                dev_warn(hsotg->dev,
3901                                         "%s: timeout GINTSTS.GINNAKEFF\n",
3902                                         __func__);
3903                }
3904        } else {
3905                /* Mask GINTSTS_GOUTNAKEFF interrupt */
3906                dwc2_hsotg_disable_gsint(hsotg, GINTSTS_GOUTNAKEFF);
3907
3908                if (!(dwc2_readl(hsotg, GINTSTS) & GINTSTS_GOUTNAKEFF))
3909                        dwc2_set_bit(hsotg, DCTL, DCTL_SGOUTNAK);
3910
3911                if (!using_dma(hsotg)) {
3912                        /* Wait for GINTSTS_RXFLVL interrupt */
3913                        if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3914                                                    GINTSTS_RXFLVL, 100)) {
3915                                dev_warn(hsotg->dev, "%s: timeout GINTSTS.RXFLVL\n",
3916                                         __func__);
3917                        } else {
3918                                /*
3919                                 * Pop GLOBAL OUT NAK status packet from RxFIFO
3920                                 * to assert GOUTNAKEFF interrupt
3921                                 */
3922                                dwc2_readl(hsotg, GRXSTSP);
3923                        }
3924                }
3925
3926                /* Wait for global nak to take effect */
3927                if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3928                                            GINTSTS_GOUTNAKEFF, 100))
3929                        dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n",
3930                                 __func__);
3931        }
3932
3933        /* Disable ep */
3934        dwc2_set_bit(hsotg, epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
3935
3936        /* Wait for ep to be disabled */
3937        if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
3938                dev_warn(hsotg->dev,
3939                         "%s: timeout DOEPCTL.EPDisable\n", __func__);
3940
3941        /* Clear EPDISBLD interrupt */
3942        dwc2_set_bit(hsotg, epint_reg, DXEPINT_EPDISBLD);
3943
3944        if (hs_ep->dir_in) {
3945                unsigned short fifo_index;
3946
3947                if (hsotg->dedicated_fifos || hs_ep->periodic)
3948                        fifo_index = hs_ep->fifo_index;
3949                else
3950                        fifo_index = 0;
3951
3952                /* Flush TX FIFO */
3953                dwc2_flush_tx_fifo(hsotg, fifo_index);
3954
3955                /* Clear Global In NP NAK in Shared FIFO for non periodic ep */
3956                if (!hsotg->dedicated_fifos && !hs_ep->periodic)
3957                        dwc2_set_bit(hsotg, DCTL, DCTL_CGNPINNAK);
3958
3959        } else {
3960                /* Remove global NAKs */
3961                dwc2_set_bit(hsotg, DCTL, DCTL_CGOUTNAK);
3962        }
3963}
3964
3965/**
3966 * dwc2_hsotg_ep_enable - enable the given endpoint
3967 * @ep: The USB endpint to configure
3968 * @desc: The USB endpoint descriptor to configure with.
3969 *
3970 * This is called from the USB gadget code's usb_ep_enable().
3971 */
3972static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
3973                                const struct usb_endpoint_descriptor *desc)
3974{
3975        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
3976        struct dwc2_hsotg *hsotg = hs_ep->parent;
3977        unsigned long flags;
3978        unsigned int index = hs_ep->index;
3979        u32 epctrl_reg;
3980        u32 epctrl;
3981        u32 mps;
3982        u32 mc;
3983        u32 mask;
3984        unsigned int dir_in;
3985        unsigned int i, val, size;
3986        int ret = 0;
3987        unsigned char ep_type;
3988        int desc_num;
3989
3990        dev_dbg(hsotg->dev,
3991                "%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n",
3992                __func__, ep->name, desc->bEndpointAddress, desc->bmAttributes,
3993                desc->wMaxPacketSize, desc->bInterval);
3994
3995        /* not to be called for EP0 */
3996        if (index == 0) {
3997                dev_err(hsotg->dev, "%s: called for EP 0\n", __func__);
3998                return -EINVAL;
3999        }
4000
4001        dir_in = (desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) ? 1 : 0;
4002        if (dir_in != hs_ep->dir_in) {
4003                dev_err(hsotg->dev, "%s: direction mismatch!\n", __func__);
4004                return -EINVAL;
4005        }
4006
4007        ep_type = desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
4008        mps = usb_endpoint_maxp(desc);
4009        mc = usb_endpoint_maxp_mult(desc);
4010
4011        /* ISOC IN in DDMA supported bInterval up to 10 */
4012        if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC &&
4013            dir_in && desc->bInterval > 10) {
4014                dev_err(hsotg->dev,
4015                        "%s: ISOC IN, DDMA: bInterval>10 not supported!\n", __func__);
4016                return -EINVAL;
4017        }
4018
4019        /* High bandwidth ISOC OUT in DDMA not supported */
4020        if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC &&
4021            !dir_in && mc > 1) {
4022                dev_err(hsotg->dev,
4023                        "%s: ISOC OUT, DDMA: HB not supported!\n", __func__);
4024                return -EINVAL;
4025        }
4026
4027        /* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */
4028
4029        epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
4030        epctrl = dwc2_readl(hsotg, epctrl_reg);
4031
4032        dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
4033                __func__, epctrl, epctrl_reg);
4034
4035        if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC)
4036                desc_num = MAX_DMA_DESC_NUM_HS_ISOC;
4037        else
4038                desc_num = MAX_DMA_DESC_NUM_GENERIC;
4039
4040        /* Allocate DMA descriptor chain for non-ctrl endpoints */
4041        if (using_desc_dma(hsotg) && !hs_ep->desc_list) {
4042                hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev,
4043                        desc_num * sizeof(struct dwc2_dma_desc),
4044                        &hs_ep->desc_list_dma, GFP_ATOMIC);
4045                if (!hs_ep->desc_list) {
4046                        ret = -ENOMEM;
4047                        goto error2;
4048                }
4049        }
4050
4051        spin_lock_irqsave(&hsotg->lock, flags);
4052
4053        epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK);
4054        epctrl |= DXEPCTL_MPS(mps);
4055
4056        /*
4057         * mark the endpoint as active, otherwise the core may ignore
4058         * transactions entirely for this endpoint
4059         */
4060        epctrl |= DXEPCTL_USBACTEP;
4061
4062        /* update the endpoint state */
4063        dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in);
4064
4065        /* default, set to non-periodic */
4066        hs_ep->isochronous = 0;
4067        hs_ep->periodic = 0;
4068        hs_ep->halted = 0;
4069        hs_ep->interval = desc->bInterval;
4070
4071        switch (ep_type) {
4072        case USB_ENDPOINT_XFER_ISOC:
4073                epctrl |= DXEPCTL_EPTYPE_ISO;
4074                epctrl |= DXEPCTL_SETEVENFR;
4075                hs_ep->isochronous = 1;
4076                hs_ep->interval = 1 << (desc->bInterval - 1);
4077                hs_ep->target_frame = TARGET_FRAME_INITIAL;
4078                hs_ep->next_desc = 0;
4079                hs_ep->compl_desc = 0;
4080                if (dir_in) {
4081                        hs_ep->periodic = 1;
4082                        mask = dwc2_readl(hsotg, DIEPMSK);
4083                        mask |= DIEPMSK_NAKMSK;
4084                        dwc2_writel(hsotg, mask, DIEPMSK);
4085                } else {
4086                        mask = dwc2_readl(hsotg, DOEPMSK);
4087                        mask |= DOEPMSK_OUTTKNEPDISMSK;
4088                        dwc2_writel(hsotg, mask, DOEPMSK);
4089                }
4090                break;
4091
4092        case USB_ENDPOINT_XFER_BULK:
4093                epctrl |= DXEPCTL_EPTYPE_BULK;
4094                break;
4095
4096        case USB_ENDPOINT_XFER_INT:
4097                if (dir_in)
4098                        hs_ep->periodic = 1;
4099
4100                if (hsotg->gadget.speed == USB_SPEED_HIGH)
4101                        hs_ep->interval = 1 << (desc->bInterval - 1);
4102
4103                epctrl |= DXEPCTL_EPTYPE_INTERRUPT;
4104                break;
4105
4106        case USB_ENDPOINT_XFER_CONTROL:
4107                epctrl |= DXEPCTL_EPTYPE_CONTROL;
4108                break;
4109        }
4110
4111        /*
4112         * if the hardware has dedicated fifos, we must give each IN EP
4113         * a unique tx-fifo even if it is non-periodic.
4114         */
4115        if (dir_in && hsotg->dedicated_fifos) {
4116                unsigned fifo_count = dwc2_hsotg_tx_fifo_count(hsotg);
4117                u32 fifo_index = 0;
4118                u32 fifo_size = UINT_MAX;
4119
4120                size = hs_ep->ep.maxpacket * hs_ep->mc;
4121                for (i = 1; i <= fifo_count; ++i) {
4122                        if (hsotg->fifo_map & (1 << i))
4123                                continue;
4124                        val = dwc2_readl(hsotg, DPTXFSIZN(i));
4125                        val = (val >> FIFOSIZE_DEPTH_SHIFT) * 4;
4126                        if (val < size)
4127                                continue;
4128                        /* Search for smallest acceptable fifo */
4129                        if (val < fifo_size) {
4130                                fifo_size = val;
4131                                fifo_index = i;
4132                        }
4133                }
4134                if (!fifo_index) {
4135                        dev_err(hsotg->dev,
4136                                "%s: No suitable fifo found\n", __func__);
4137                        ret = -ENOMEM;
4138                        goto error1;
4139                }
4140                epctrl &= ~(DXEPCTL_TXFNUM_LIMIT << DXEPCTL_TXFNUM_SHIFT);
4141                hsotg->fifo_map |= 1 << fifo_index;
4142                epctrl |= DXEPCTL_TXFNUM(fifo_index);
4143                hs_ep->fifo_index = fifo_index;
4144                hs_ep->fifo_size = fifo_size;
4145        }
4146
4147        /* for non control endpoints, set PID to D0 */
4148        if (index && !hs_ep->isochronous)
4149                epctrl |= DXEPCTL_SETD0PID;
4150
4151        /* WA for Full speed ISOC IN in DDMA mode.
4152         * By Clear NAK status of EP, core will send ZLP
4153         * to IN token and assert NAK interrupt relying
4154         * on TxFIFO status only
4155         */
4156
4157        if (hsotg->gadget.speed == USB_SPEED_FULL &&
4158            hs_ep->isochronous && dir_in) {
4159                /* The WA applies only to core versions from 2.72a
4160                 * to 4.00a (including both). Also for FS_IOT_1.00a
4161                 * and HS_IOT_1.00a.
4162                 */
4163                u32 gsnpsid = dwc2_readl(hsotg, GSNPSID);
4164
4165                if ((gsnpsid >= DWC2_CORE_REV_2_72a &&
4166                     gsnpsid <= DWC2_CORE_REV_4_00a) ||
4167                     gsnpsid == DWC2_FS_IOT_REV_1_00a ||
4168                     gsnpsid == DWC2_HS_IOT_REV_1_00a)
4169                        epctrl |= DXEPCTL_CNAK;
4170        }
4171
4172        dev_dbg(hsotg->dev, "%s: write DxEPCTL=0x%08x\n",
4173                __func__, epctrl);
4174
4175        dwc2_writel(hsotg, epctrl, epctrl_reg);
4176        dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x\n",
4177                __func__, dwc2_readl(hsotg, epctrl_reg));
4178
4179        /* enable the endpoint interrupt */
4180        dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1);
4181
4182error1:
4183        spin_unlock_irqrestore(&hsotg->lock, flags);
4184
4185error2:
4186        if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
4187                dmam_free_coherent(hsotg->dev, desc_num *
4188                        sizeof(struct dwc2_dma_desc),
4189                        hs_ep->desc_list, hs_ep->desc_list_dma);
4190                hs_ep->desc_list = NULL;
4191        }
4192
4193        return ret;
4194}
4195
4196/**
4197 * dwc2_hsotg_ep_disable - disable given endpoint
4198 * @ep: The endpoint to disable.
4199 */
4200static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
4201{
4202        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4203        struct dwc2_hsotg *hsotg = hs_ep->parent;
4204        int dir_in = hs_ep->dir_in;
4205        int index = hs_ep->index;
4206        u32 epctrl_reg;
4207        u32 ctrl;
4208
4209        dev_dbg(hsotg->dev, "%s(ep %p)\n", __func__, ep);
4210
4211        if (ep == &hsotg->eps_out[0]->ep) {
4212                dev_err(hsotg->dev, "%s: called for ep0\n", __func__);
4213                return -EINVAL;
4214        }
4215
4216        if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
4217                dev_err(hsotg->dev, "%s: called in host mode?\n", __func__);
4218                return -EINVAL;
4219        }
4220
4221        epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
4222
4223        ctrl = dwc2_readl(hsotg, epctrl_reg);
4224
4225        if (ctrl & DXEPCTL_EPENA)
4226                dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
4227
4228        ctrl &= ~DXEPCTL_EPENA;
4229        ctrl &= ~DXEPCTL_USBACTEP;
4230        ctrl |= DXEPCTL_SNAK;
4231
4232        dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
4233        dwc2_writel(hsotg, ctrl, epctrl_reg);
4234
4235        /* disable endpoint interrupts */
4236        dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);
4237
4238        /* terminate all requests with shutdown */
4239        kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);
4240
4241        hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
4242        hs_ep->fifo_index = 0;
4243        hs_ep->fifo_size = 0;
4244
4245        return 0;
4246}
4247
4248static int dwc2_hsotg_ep_disable_lock(struct usb_ep *ep)
4249{
4250        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4251        struct dwc2_hsotg *hsotg = hs_ep->parent;
4252        unsigned long flags;
4253        int ret;
4254
4255        spin_lock_irqsave(&hsotg->lock, flags);
4256        ret = dwc2_hsotg_ep_disable(ep);
4257        spin_unlock_irqrestore(&hsotg->lock, flags);
4258        return ret;
4259}
4260
4261/**
4262 * on_list - check request is on the given endpoint
4263 * @ep: The endpoint to check.
4264 * @test: The request to test if it is on the endpoint.
4265 */
4266static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test)
4267{
4268        struct dwc2_hsotg_req *req, *treq;
4269
4270        list_for_each_entry_safe(req, treq, &ep->queue, queue) {
4271                if (req == test)
4272                        return true;
4273        }
4274
4275        return false;
4276}
4277
4278/**
4279 * dwc2_hsotg_ep_dequeue - dequeue given endpoint
4280 * @ep: The endpoint to dequeue.
4281 * @req: The request to be removed from a queue.
4282 */
4283static int dwc2_hsotg_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
4284{
4285        struct dwc2_hsotg_req *hs_req = our_req(req);
4286        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4287        struct dwc2_hsotg *hs = hs_ep->parent;
4288        unsigned long flags;
4289
4290        dev_dbg(hs->dev, "ep_dequeue(%p,%p)\n", ep, req);
4291
4292        spin_lock_irqsave(&hs->lock, flags);
4293
4294        if (!on_list(hs_ep, hs_req)) {
4295                spin_unlock_irqrestore(&hs->lock, flags);
4296                return -EINVAL;
4297        }
4298
4299        /* Dequeue already started request */
4300        if (req == &hs_ep->req->req)
4301                dwc2_hsotg_ep_stop_xfr(hs, hs_ep);
4302
4303        dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);
4304        spin_unlock_irqrestore(&hs->lock, flags);
4305
4306        return 0;
4307}
4308
4309/**
4310 * dwc2_hsotg_ep_sethalt - set halt on a given endpoint
4311 * @ep: The endpoint to set halt.
4312 * @value: Set or unset the halt.
4313 * @now: If true, stall the endpoint now. Otherwise return -EAGAIN if
4314 *       the endpoint is busy processing requests.
4315 *
4316 * We need to stall the endpoint immediately if request comes from set_feature
4317 * protocol command handler.
4318 */
4319static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now)
4320{
4321        struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4322        struct dwc2_hsotg *hs = hs_ep->parent;
4323        int index = hs_ep->index;
4324        u32 epreg;
4325        u32 epctl;
4326        u32 xfertype;
4327
4328        dev_info(hs->dev, "%s(ep %p %s, %d)\n", __func__, ep, ep->name, value);
4329
4330        if (index == 0) {
4331                if (value)
4332                        dwc2_hsotg_stall_ep0(hs);
4333                else
4334                        dev_warn(hs->dev,
4335                                 "%s: can't clear halt on ep0\n", __func__);
4336                return 0;
4337        }
4338
4339        if (hs_ep->isochronous) {
4340                dev_err(hs->dev, "%s is Isochronous Endpoint\n", ep->name);
4341                return -EINVAL;
4342        }
4343
4344        if (!now && value && !list_empty(&hs_ep->queue)) {
4345                dev_dbg(hs->dev, "%s request is pending, cannot halt\n",
4346                        ep->name);
4347                return -EAGAIN;
4348        }
4349
4350        if (hs_ep->dir_in) {
4351                epreg = DIEPCTL(index);
4352                epctl = dwc2_readl(hs, epreg);
4353
4354                if (value) {
4355                        epctl |= DXEPCTL_STALL | DXEPCTL_SNAK;
4356                        if (epctl & DXEPCTL_EPENA)
4357