linux/drivers/usb/cdns3/cdns3-gadget.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence USBSS DRD Driver - gadget side.
   4 *
   5 * Copyright (C) 2018-2019 Cadence Design Systems.
   6 * Copyright (C) 2017-2018 NXP
   7 *
   8 * Authors: Pawel Jez <pjez@cadence.com>,
   9 *          Pawel Laszczak <pawell@cadence.com>
  10 *          Peter Chen <peter.chen@nxp.com>
  11 */
  12
  13/*
  14 * Work around 1:
  15 * At some situations, the controller may get stale data address in TRB
  16 * at below sequences:
  17 * 1. Controller read TRB includes data address
  18 * 2. Software updates TRBs includes data address and Cycle bit
  19 * 3. Controller read TRB which includes Cycle bit
  20 * 4. DMA run with stale data address
  21 *
  22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23 * After preparing all TRBs driver needs to check the position of DMA and
  24 * if the DMA point to the first just added TRB and doorbell is 1,
  25 * then driver must defer making this TRB as valid. This TRB will be make
  26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27 * interrupt.
  28 *
  29 * Issue has been fixed in DEV_VER_V3 version of controller.
  30 *
  31 * Work around 2:
  32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34 * in correct order. If the first packet in the buffer will not be handled,
  35 * then the following packets directed for other endpoints and  functions
  36 * will be blocked.
  37 * Additionally the packets directed to one endpoint can block entire on-chip
  38 * buffers. In this case transfer to other endpoints also will blocked.
  39 *
  40 * To resolve this issue after raising the descriptor missing interrupt
  41 * driver prepares internal usb_request object and use it to arm DMA transfer.
  42 *
  43 * The problematic situation was observed in case when endpoint has been enabled
  44 * but no usb_request were queued. Driver try detects such endpoints and will
  45 * use this workaround only for these endpoint.
  46 *
  47 * Driver use limited number of buffer. This number can be set by macro
  48 * CDNS3_WA2_NUM_BUFFERS.
  49 *
  50 * Such blocking situation was observed on ACM gadget. For this function
  51 * host send OUT data packet but ACM function is not prepared for this packet.
  52 * It's cause that buffer placed in on chip memory block transfer to other
  53 * endpoints.
  54 *
  55 * Issue has been fixed in DEV_VER_V2 version of controller.
  56 *
  57 */
  58
  59#include <linux/dma-mapping.h>
  60#include <linux/usb/gadget.h>
  61#include <linux/module.h>
  62#include <linux/dmapool.h>
  63#include <linux/iopoll.h>
  64
  65#include "core.h"
  66#include "gadget-export.h"
  67#include "cdns3-gadget.h"
  68#include "cdns3-trace.h"
  69#include "drd.h"
  70
  71static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  72                                   struct usb_request *request,
  73                                   gfp_t gfp_flags);
  74
  75static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
  76                                 struct usb_request *request);
  77
  78static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
  79                                        struct usb_request *request);
  80
  81/**
  82 * cdns3_clear_register_bit - clear bit in given register.
  83 * @ptr: address of device controller register to be read and changed
  84 * @mask: bits requested to clar
  85 */
  86static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
  87{
  88        mask = readl(ptr) & ~mask;
  89        writel(mask, ptr);
  90}
  91
  92/**
  93 * cdns3_set_register_bit - set bit in given register.
  94 * @ptr: address of device controller register to be read and changed
  95 * @mask: bits requested to set
  96 */
  97void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  98{
  99        mask = readl(ptr) | mask;
 100        writel(mask, ptr);
 101}
 102
 103/**
 104 * cdns3_ep_addr_to_index - Macro converts endpoint address to
 105 * index of endpoint object in cdns3_device.eps[] container
 106 * @ep_addr: endpoint address for which endpoint object is required
 107 *
 108 */
 109u8 cdns3_ep_addr_to_index(u8 ep_addr)
 110{
 111        return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 112}
 113
 114static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 115                             struct cdns3_endpoint *priv_ep)
 116{
 117        int dma_index;
 118
 119        dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 120
 121        return dma_index / TRB_SIZE;
 122}
 123
 124/**
 125 * cdns3_next_request - returns next request from list
 126 * @list: list containing requests
 127 *
 128 * Returns request or NULL if no requests in list
 129 */
 130struct usb_request *cdns3_next_request(struct list_head *list)
 131{
 132        return list_first_entry_or_null(list, struct usb_request, list);
 133}
 134
 135/**
 136 * cdns3_next_align_buf - returns next buffer from list
 137 * @list: list containing buffers
 138 *
 139 * Returns buffer or NULL if no buffers in list
 140 */
 141static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 142{
 143        return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 144}
 145
 146/**
 147 * cdns3_next_priv_request - returns next request from list
 148 * @list: list containing requests
 149 *
 150 * Returns request or NULL if no requests in list
 151 */
 152static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 153{
 154        return list_first_entry_or_null(list, struct cdns3_request, list);
 155}
 156
 157/**
 158 * cdns3_select_ep - selects endpoint
 159 * @priv_dev:  extended gadget object
 160 * @ep: endpoint address
 161 */
 162void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 163{
 164        if (priv_dev->selected_ep == ep)
 165                return;
 166
 167        priv_dev->selected_ep = ep;
 168        writel(ep, &priv_dev->regs->ep_sel);
 169}
 170
 171/**
 172 * cdns3_get_tdl - gets current tdl for selected endpoint.
 173 * @priv_dev:  extended gadget object
 174 *
 175 * Before calling this function the appropriate endpoint must
 176 * be selected by means of cdns3_select_ep function.
 177 */
 178static int cdns3_get_tdl(struct cdns3_device *priv_dev)
 179{
 180        if (priv_dev->dev_ver < DEV_VER_V3)
 181                return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 182        else
 183                return readl(&priv_dev->regs->ep_tdl);
 184}
 185
 186dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 187                                 struct cdns3_trb *trb)
 188{
 189        u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 190
 191        return priv_ep->trb_pool_dma + offset;
 192}
 193
 194static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 195{
 196        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 197
 198        if (priv_ep->trb_pool) {
 199                dma_pool_free(priv_dev->eps_dma_pool,
 200                              priv_ep->trb_pool, priv_ep->trb_pool_dma);
 201                priv_ep->trb_pool = NULL;
 202        }
 203}
 204
 205/**
 206 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 207 * @priv_ep:  endpoint object
 208 *
 209 * Function will return 0 on success or -ENOMEM on allocation error
 210 */
 211int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 212{
 213        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 214        int ring_size = TRB_RING_SIZE;
 215        int num_trbs = ring_size / TRB_SIZE;
 216        struct cdns3_trb *link_trb;
 217
 218        if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
 219                cdns3_free_trb_pool(priv_ep);
 220
 221        if (!priv_ep->trb_pool) {
 222                priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
 223                                                   GFP_DMA32 | GFP_ATOMIC,
 224                                                   &priv_ep->trb_pool_dma);
 225
 226                if (!priv_ep->trb_pool)
 227                        return -ENOMEM;
 228
 229                priv_ep->alloc_ring_size = ring_size;
 230        }
 231
 232        memset(priv_ep->trb_pool, 0, ring_size);
 233
 234        priv_ep->num_trbs = num_trbs;
 235
 236        if (!priv_ep->num)
 237                return 0;
 238
 239        /* Initialize the last TRB as Link TRB */
 240        link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 241
 242        if (priv_ep->use_streams) {
 243                /*
 244                 * For stream capable endpoints driver use single correct TRB.
 245                 * The last trb has zeroed cycle bit
 246                 */
 247                link_trb->control = 0;
 248        } else {
 249                link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
 250                link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
 251        }
 252        return 0;
 253}
 254
 255/**
 256 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 257 * @priv_ep: endpoint object
 258 *
 259 * Endpoint must be selected before call to this function
 260 */
 261static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 262{
 263        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 264        int val;
 265
 266        trace_cdns3_halt(priv_ep, 1, 1);
 267
 268        writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 269               &priv_dev->regs->ep_cmd);
 270
 271        /* wait for DFLUSH cleared */
 272        readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 273                                  !(val & EP_CMD_DFLUSH), 1, 1000);
 274        priv_ep->flags |= EP_STALLED;
 275        priv_ep->flags &= ~EP_STALL_PENDING;
 276}
 277
 278/**
 279 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 280 * @priv_dev: extended gadget object
 281 */
 282void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 283{
 284        int i;
 285
 286        writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 287
 288        cdns3_allow_enable_l1(priv_dev, 0);
 289        priv_dev->hw_configured_flag = 0;
 290        priv_dev->onchip_used_size = 0;
 291        priv_dev->out_mem_is_allocated = 0;
 292        priv_dev->wait_for_setup = 0;
 293        priv_dev->using_streams = 0;
 294
 295        for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
 296                if (priv_dev->eps[i])
 297                        priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
 298}
 299
 300/**
 301 * cdns3_ep_inc_trb - increment a trb index.
 302 * @index: Pointer to the TRB index to increment.
 303 * @cs: Cycle state
 304 * @trb_in_seg: number of TRBs in segment
 305 *
 306 * The index should never point to the link TRB. After incrementing,
 307 * if it is point to the link TRB, wrap around to the beginning and revert
 308 * cycle state bit The
 309 * link TRB is always at the last TRB entry.
 310 */
 311static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 312{
 313        (*index)++;
 314        if (*index == (trb_in_seg - 1)) {
 315                *index = 0;
 316                *cs ^=  1;
 317        }
 318}
 319
 320/**
 321 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 322 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 323 */
 324static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 325{
 326        priv_ep->free_trbs--;
 327        cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 328}
 329
 330/**
 331 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 332 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 333 */
 334static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 335{
 336        priv_ep->free_trbs++;
 337        cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 338}
 339
 340static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 341{
 342        struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 343        int current_trb = priv_req->start_trb;
 344
 345        while (current_trb != priv_req->end_trb) {
 346                cdns3_ep_inc_deq(priv_ep);
 347                current_trb = priv_ep->dequeue;
 348        }
 349
 350        cdns3_ep_inc_deq(priv_ep);
 351}
 352
 353/**
 354 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 355 * @priv_dev: Extended gadget object
 356 * @enable: Enable/disable permit to transition to L1.
 357 *
 358 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 359 * then controller answer with ACK handshake.
 360 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 361 * then controller answer with NYET handshake.
 362 */
 363void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 364{
 365        if (enable)
 366                writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 367        else
 368                writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 369}
 370
 371enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 372{
 373        u32 reg;
 374
 375        reg = readl(&priv_dev->regs->usb_sts);
 376
 377        if (DEV_SUPERSPEED(reg))
 378                return USB_SPEED_SUPER;
 379        else if (DEV_HIGHSPEED(reg))
 380                return USB_SPEED_HIGH;
 381        else if (DEV_FULLSPEED(reg))
 382                return USB_SPEED_FULL;
 383        else if (DEV_LOWSPEED(reg))
 384                return USB_SPEED_LOW;
 385        return USB_SPEED_UNKNOWN;
 386}
 387
 388/**
 389 * cdns3_start_all_request - add to ring all request not started
 390 * @priv_dev: Extended gadget object
 391 * @priv_ep: The endpoint for whom request will be started.
 392 *
 393 * Returns return ENOMEM if transfer ring i not enough TRBs to start
 394 *         all requests.
 395 */
 396static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 397                                   struct cdns3_endpoint *priv_ep)
 398{
 399        struct usb_request *request;
 400        int ret = 0;
 401        u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 402
 403        /*
 404         * If the last pending transfer is INTERNAL
 405         * OR streams are enabled for this endpoint
 406         * do NOT start new transfer till the last one is pending
 407         */
 408        if (!pending_empty) {
 409                struct cdns3_request *priv_req;
 410
 411                request = cdns3_next_request(&priv_ep->pending_req_list);
 412                priv_req = to_cdns3_request(request);
 413                if ((priv_req->flags & REQUEST_INTERNAL) ||
 414                    (priv_ep->flags & EP_TDLCHK_EN) ||
 415                        priv_ep->use_streams) {
 416                        dev_dbg(priv_dev->dev, "Blocking external request\n");
 417                        return ret;
 418                }
 419        }
 420
 421        while (!list_empty(&priv_ep->deferred_req_list)) {
 422                request = cdns3_next_request(&priv_ep->deferred_req_list);
 423
 424                if (!priv_ep->use_streams) {
 425                        ret = cdns3_ep_run_transfer(priv_ep, request);
 426                } else {
 427                        priv_ep->stream_sg_idx = 0;
 428                        ret = cdns3_ep_run_stream_transfer(priv_ep, request);
 429                }
 430                if (ret)
 431                        return ret;
 432
 433                list_move_tail(&request->list, &priv_ep->pending_req_list);
 434                if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
 435                        break;
 436        }
 437
 438        priv_ep->flags &= ~EP_RING_FULL;
 439        return ret;
 440}
 441
 442/*
 443 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 444 * driver try to detect whether endpoint need additional internal
 445 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 446 * if before first DESCMISS interrupt the DMA will be armed.
 447 */
 448#define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
 449        if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 450                priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 451                (reg) |= EP_STS_EN_DESCMISEN; \
 452        } } while (0)
 453
 454static void __cdns3_descmiss_copy_data(struct usb_request *request,
 455        struct usb_request *descmiss_req)
 456{
 457        int length = request->actual + descmiss_req->actual;
 458        struct scatterlist *s = request->sg;
 459
 460        if (!s) {
 461                if (length <= request->length) {
 462                        memcpy(&((u8 *)request->buf)[request->actual],
 463                               descmiss_req->buf,
 464                               descmiss_req->actual);
 465                        request->actual = length;
 466                } else {
 467                        /* It should never occures */
 468                        request->status = -ENOMEM;
 469                }
 470        } else {
 471                if (length <= sg_dma_len(s)) {
 472                        void *p = phys_to_virt(sg_dma_address(s));
 473
 474                        memcpy(&((u8 *)p)[request->actual],
 475                                descmiss_req->buf,
 476                                descmiss_req->actual);
 477                        request->actual = length;
 478                } else {
 479                        request->status = -ENOMEM;
 480                }
 481        }
 482}
 483
 484/**
 485 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
 486 * request queued by class driver.
 487 * @priv_ep: extended endpoint object
 488 * @request: request object
 489 */
 490static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 491                                         struct usb_request *request)
 492{
 493        struct usb_request *descmiss_req;
 494        struct cdns3_request *descmiss_priv_req;
 495
 496        while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 497                int chunk_end;
 498
 499                descmiss_priv_req =
 500                        cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 501                descmiss_req = &descmiss_priv_req->request;
 502
 503                /* driver can't touch pending request */
 504                if (descmiss_priv_req->flags & REQUEST_PENDING)
 505                        break;
 506
 507                chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 508                request->status = descmiss_req->status;
 509                __cdns3_descmiss_copy_data(request, descmiss_req);
 510                list_del_init(&descmiss_priv_req->list);
 511                kfree(descmiss_req->buf);
 512                cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 513                --priv_ep->wa2_counter;
 514
 515                if (!chunk_end)
 516                        break;
 517        }
 518}
 519
 520static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 521                                                     struct cdns3_endpoint *priv_ep,
 522                                                     struct cdns3_request *priv_req)
 523{
 524        if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 525            priv_req->flags & REQUEST_INTERNAL) {
 526                struct usb_request *req;
 527
 528                req = cdns3_next_request(&priv_ep->deferred_req_list);
 529
 530                priv_ep->descmis_req = NULL;
 531
 532                if (!req)
 533                        return NULL;
 534
 535                /* unmap the gadget request before copying data */
 536                usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
 537                                                priv_ep->dir);
 538
 539                cdns3_wa2_descmiss_copy_data(priv_ep, req);
 540                if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 541                    req->length != req->actual) {
 542                        /* wait for next part of transfer */
 543                        /* re-map the gadget request buffer*/
 544                        usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
 545                                usb_endpoint_dir_in(priv_ep->endpoint.desc));
 546                        return NULL;
 547                }
 548
 549                if (req->status == -EINPROGRESS)
 550                        req->status = 0;
 551
 552                list_del_init(&req->list);
 553                cdns3_start_all_request(priv_dev, priv_ep);
 554                return req;
 555        }
 556
 557        return &priv_req->request;
 558}
 559
 560static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 561                                     struct cdns3_endpoint *priv_ep,
 562                                     struct cdns3_request *priv_req)
 563{
 564        int deferred = 0;
 565
 566        /*
 567         * If transfer was queued before DESCMISS appear than we
 568         * can disable handling of DESCMISS interrupt. Driver assumes that it
 569         * can disable special treatment for this endpoint.
 570         */
 571        if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 572                u32 reg;
 573
 574                cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 575                priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 576                reg = readl(&priv_dev->regs->ep_sts_en);
 577                reg &= ~EP_STS_EN_DESCMISEN;
 578                trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 579                writel(reg, &priv_dev->regs->ep_sts_en);
 580        }
 581
 582        if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 583                u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 584                u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 585
 586                /*
 587                 *  DESCMISS transfer has been finished, so data will be
 588                 *  directly copied from internal allocated usb_request
 589                 *  objects.
 590                 */
 591                if (pending_empty && !descmiss_empty &&
 592                    !(priv_req->flags & REQUEST_INTERNAL)) {
 593                        cdns3_wa2_descmiss_copy_data(priv_ep,
 594                                                     &priv_req->request);
 595
 596                        trace_cdns3_wa2(priv_ep, "get internal stored data");
 597
 598                        list_add_tail(&priv_req->request.list,
 599                                      &priv_ep->pending_req_list);
 600                        cdns3_gadget_giveback(priv_ep, priv_req,
 601                                              priv_req->request.status);
 602
 603                        /*
 604                         * Intentionally driver returns positive value as
 605                         * correct value. It informs that transfer has
 606                         * been finished.
 607                         */
 608                        return EINPROGRESS;
 609                }
 610
 611                /*
 612                 * Driver will wait for completion DESCMISS transfer,
 613                 * before starts new, not DESCMISS transfer.
 614                 */
 615                if (!pending_empty && !descmiss_empty) {
 616                        trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 617                        deferred = 1;
 618                }
 619
 620                if (priv_req->flags & REQUEST_INTERNAL)
 621                        list_add_tail(&priv_req->list,
 622                                      &priv_ep->wa2_descmiss_req_list);
 623        }
 624
 625        return deferred;
 626}
 627
 628static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 629{
 630        struct cdns3_request *priv_req;
 631
 632        while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 633                u8 chain;
 634
 635                priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 636                chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 637
 638                trace_cdns3_wa2(priv_ep, "removes eldest request");
 639
 640                kfree(priv_req->request.buf);
 641                cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 642                                             &priv_req->request);
 643                list_del_init(&priv_req->list);
 644                --priv_ep->wa2_counter;
 645
 646                if (!chain)
 647                        break;
 648        }
 649}
 650
 651/**
 652 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 653 * @priv_ep: extended gadget object
 654 *
 655 * This function is used only for WA2. For more information see Work around 2
 656 * description.
 657 */
 658static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 659{
 660        struct cdns3_request *priv_req;
 661        struct usb_request *request;
 662        u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 663
 664        /* check for pending transfer */
 665        if (!pending_empty) {
 666                trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
 667                return;
 668        }
 669
 670        if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 671                priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 672                priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 673        }
 674
 675        trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 676
 677        if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
 678                trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
 679                cdns3_wa2_remove_old_request(priv_ep);
 680        }
 681
 682        request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 683                                                GFP_ATOMIC);
 684        if (!request)
 685                goto err;
 686
 687        priv_req = to_cdns3_request(request);
 688        priv_req->flags |= REQUEST_INTERNAL;
 689
 690        /* if this field is still assigned it indicate that transfer related
 691         * with this request has not been finished yet. Driver in this
 692         * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 693         * flag to previous one. It will indicate that current request is
 694         * part of the previous one.
 695         */
 696        if (priv_ep->descmis_req)
 697                priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 698
 699        priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 700                                        GFP_ATOMIC);
 701        priv_ep->wa2_counter++;
 702
 703        if (!priv_req->request.buf) {
 704                cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 705                goto err;
 706        }
 707
 708        priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 709        priv_ep->descmis_req = priv_req;
 710
 711        __cdns3_gadget_ep_queue(&priv_ep->endpoint,
 712                                &priv_ep->descmis_req->request,
 713                                GFP_ATOMIC);
 714
 715        return;
 716
 717err:
 718        dev_err(priv_ep->cdns3_dev->dev,
 719                "Failed: No sufficient memory for DESCMIS\n");
 720}
 721
 722static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
 723{
 724        u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 725
 726        if (tdl) {
 727                u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
 728
 729                writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
 730                       &priv_dev->regs->ep_cmd);
 731        }
 732}
 733
 734static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
 735{
 736        u32 ep_sts_reg;
 737
 738        /* select EP0-out */
 739        cdns3_select_ep(priv_dev, 0);
 740
 741        ep_sts_reg = readl(&priv_dev->regs->ep_sts);
 742
 743        if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
 744                u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
 745                struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
 746
 747                if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
 748                    outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
 749                        u8 pending_empty = list_empty(&outq_ep->pending_req_list);
 750
 751                        if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
 752                            (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
 753                            !pending_empty) {
 754                        } else {
 755                                u32 ep_sts_en_reg;
 756                                u32 ep_cmd_reg;
 757
 758                                cdns3_select_ep(priv_dev, outq_ep->num |
 759                                                outq_ep->dir);
 760                                ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
 761                                ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
 762
 763                                outq_ep->flags |= EP_TDLCHK_EN;
 764                                cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 765                                                       EP_CFG_TDL_CHK);
 766
 767                                cdns3_wa2_enable_detection(priv_dev, outq_ep,
 768                                                           ep_sts_en_reg);
 769                                writel(ep_sts_en_reg,
 770                                       &priv_dev->regs->ep_sts_en);
 771                                /* reset tdl value to zero */
 772                                cdns3_wa2_reset_tdl(priv_dev);
 773                                /*
 774                                 * Memory barrier - Reset tdl before ringing the
 775                                 * doorbell.
 776                                 */
 777                                wmb();
 778                                if (EP_CMD_DRDY & ep_cmd_reg) {
 779                                        trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
 780
 781                                } else {
 782                                        trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
 783                                        /*
 784                                         * ring doorbell to generate DESCMIS irq
 785                                         */
 786                                        writel(EP_CMD_DRDY,
 787                                               &priv_dev->regs->ep_cmd);
 788                                }
 789                        }
 790                }
 791        }
 792}
 793
 794/**
 795 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 796 * @priv_ep: The endpoint to whom the request belongs to
 797 * @priv_req: The request we're giving back
 798 * @status: completion code for the request
 799 *
 800 * Must be called with controller's lock held and interrupts disabled. This
 801 * function will unmap @req and call its ->complete() callback to notify upper
 802 * layers that it has completed.
 803 */
 804void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 805                           struct cdns3_request *priv_req,
 806                           int status)
 807{
 808        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 809        struct usb_request *request = &priv_req->request;
 810
 811        list_del_init(&request->list);
 812
 813        if (request->status == -EINPROGRESS)
 814                request->status = status;
 815
 816        usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 817                                        priv_ep->dir);
 818
 819        if ((priv_req->flags & REQUEST_UNALIGNED) &&
 820            priv_ep->dir == USB_DIR_OUT && !request->status) {
 821                /* Make DMA buffer CPU accessible */
 822                dma_sync_single_for_cpu(priv_dev->sysdev,
 823                        priv_req->aligned_buf->dma,
 824                        priv_req->aligned_buf->size,
 825                        priv_req->aligned_buf->dir);
 826                memcpy(request->buf, priv_req->aligned_buf->buf,
 827                       request->length);
 828        }
 829
 830        priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 831        /* All TRBs have finished, clear the counter */
 832        priv_req->finished_trb = 0;
 833        trace_cdns3_gadget_giveback(priv_req);
 834
 835        if (priv_dev->dev_ver < DEV_VER_V2) {
 836                request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 837                                                    priv_req);
 838                if (!request)
 839                        return;
 840        }
 841
 842        if (request->complete) {
 843                spin_unlock(&priv_dev->lock);
 844                usb_gadget_giveback_request(&priv_ep->endpoint,
 845                                            request);
 846                spin_lock(&priv_dev->lock);
 847        }
 848
 849        if (request->buf == priv_dev->zlp_buf)
 850                cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 851}
 852
 853static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 854{
 855        /* Work around for stale data address in TRB*/
 856        if (priv_ep->wa1_set) {
 857                trace_cdns3_wa1(priv_ep, "restore cycle bit");
 858
 859                priv_ep->wa1_set = 0;
 860                priv_ep->wa1_trb_index = 0xFFFF;
 861                if (priv_ep->wa1_cycle_bit) {
 862                        priv_ep->wa1_trb->control =
 863                                priv_ep->wa1_trb->control | cpu_to_le32(0x1);
 864                } else {
 865                        priv_ep->wa1_trb->control =
 866                                priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
 867                }
 868        }
 869}
 870
 871static void cdns3_free_aligned_request_buf(struct work_struct *work)
 872{
 873        struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 874                                        aligned_buf_wq);
 875        struct cdns3_aligned_buf *buf, *tmp;
 876        unsigned long flags;
 877
 878        spin_lock_irqsave(&priv_dev->lock, flags);
 879
 880        list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 881                if (!buf->in_use) {
 882                        list_del(&buf->list);
 883
 884                        /*
 885                         * Re-enable interrupts to free DMA capable memory.
 886                         * Driver can't free this memory with disabled
 887                         * interrupts.
 888                         */
 889                        spin_unlock_irqrestore(&priv_dev->lock, flags);
 890                        dma_free_noncoherent(priv_dev->sysdev, buf->size,
 891                                          buf->buf, buf->dma, buf->dir);
 892                        kfree(buf);
 893                        spin_lock_irqsave(&priv_dev->lock, flags);
 894                }
 895        }
 896
 897        spin_unlock_irqrestore(&priv_dev->lock, flags);
 898}
 899
 900static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 901{
 902        struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 903        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 904        struct cdns3_aligned_buf *buf;
 905
 906        /* check if buffer is aligned to 8. */
 907        if (!((uintptr_t)priv_req->request.buf & 0x7))
 908                return 0;
 909
 910        buf = priv_req->aligned_buf;
 911
 912        if (!buf || priv_req->request.length > buf->size) {
 913                buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 914                if (!buf)
 915                        return -ENOMEM;
 916
 917                buf->size = priv_req->request.length;
 918                buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
 919                        DMA_TO_DEVICE : DMA_FROM_DEVICE;
 920
 921                buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,
 922                                              buf->size,
 923                                              &buf->dma,
 924                                              buf->dir,
 925                                              GFP_ATOMIC);
 926                if (!buf->buf) {
 927                        kfree(buf);
 928                        return -ENOMEM;
 929                }
 930
 931                if (priv_req->aligned_buf) {
 932                        trace_cdns3_free_aligned_request(priv_req);
 933                        priv_req->aligned_buf->in_use = 0;
 934                        queue_work(system_freezable_wq,
 935                                   &priv_dev->aligned_buf_wq);
 936                }
 937
 938                buf->in_use = 1;
 939                priv_req->aligned_buf = buf;
 940
 941                list_add_tail(&buf->list,
 942                              &priv_dev->aligned_buf_list);
 943        }
 944
 945        if (priv_ep->dir == USB_DIR_IN) {
 946                /* Make DMA buffer CPU accessible */
 947                dma_sync_single_for_cpu(priv_dev->sysdev,
 948                        buf->dma, buf->size, buf->dir);
 949                memcpy(buf->buf, priv_req->request.buf,
 950                       priv_req->request.length);
 951        }
 952
 953        /* Transfer DMA buffer ownership back to device */
 954        dma_sync_single_for_device(priv_dev->sysdev,
 955                        buf->dma, buf->size, buf->dir);
 956
 957        priv_req->flags |= REQUEST_UNALIGNED;
 958        trace_cdns3_prepare_aligned_request(priv_req);
 959
 960        return 0;
 961}
 962
 963static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 964                                  struct cdns3_trb *trb)
 965{
 966        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 967
 968        if (!priv_ep->wa1_set) {
 969                u32 doorbell;
 970
 971                doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 972
 973                if (doorbell) {
 974                        priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 975                        priv_ep->wa1_set = 1;
 976                        priv_ep->wa1_trb = trb;
 977                        priv_ep->wa1_trb_index = priv_ep->enqueue;
 978                        trace_cdns3_wa1(priv_ep, "set guard");
 979                        return 0;
 980                }
 981        }
 982        return 1;
 983}
 984
 985static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 986                                             struct cdns3_endpoint *priv_ep)
 987{
 988        int dma_index;
 989        u32 doorbell;
 990
 991        doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 992        dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 993
 994        if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 995                cdns3_wa1_restore_cycle_bit(priv_ep);
 996}
 997
 998static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
 999                                        struct usb_request *request)
1000{
1001        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1002        struct cdns3_request *priv_req;
1003        struct cdns3_trb *trb;
1004        dma_addr_t trb_dma;
1005        int address;
1006        u32 control;
1007        u32 length;
1008        u32 tdl;
1009        unsigned int sg_idx = priv_ep->stream_sg_idx;
1010
1011        priv_req = to_cdns3_request(request);
1012        address = priv_ep->endpoint.desc->bEndpointAddress;
1013
1014        priv_ep->flags |= EP_PENDING_REQUEST;
1015
1016        /* must allocate buffer aligned to 8 */
1017        if (priv_req->flags & REQUEST_UNALIGNED)
1018                trb_dma = priv_req->aligned_buf->dma;
1019        else
1020                trb_dma = request->dma;
1021
1022        /*  For stream capable endpoints driver use only single TD. */
1023        trb = priv_ep->trb_pool + priv_ep->enqueue;
1024        priv_req->start_trb = priv_ep->enqueue;
1025        priv_req->end_trb = priv_req->start_trb;
1026        priv_req->trb = trb;
1027
1028        cdns3_select_ep(priv_ep->cdns3_dev, address);
1029
1030        control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1031                  TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1032
1033        if (!request->num_sgs) {
1034                trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1035                length = request->length;
1036        } else {
1037                trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1038                length = request->sg[sg_idx].length;
1039        }
1040
1041        tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1042
1043        trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1044
1045        /*
1046         * For DEV_VER_V2 controller version we have enabled
1047         * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1048         * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1049         */
1050        if (priv_dev->dev_ver >= DEV_VER_V2) {
1051                if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1052                        trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1053        }
1054        priv_req->flags |= REQUEST_PENDING;
1055
1056        trb->control = cpu_to_le32(control);
1057
1058        trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1059
1060        /*
1061         * Memory barrier - Cycle Bit must be set before trb->length  and
1062         * trb->buffer fields.
1063         */
1064        wmb();
1065
1066        /* always first element */
1067        writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1068               &priv_dev->regs->ep_traddr);
1069
1070        if (!(priv_ep->flags & EP_STALLED)) {
1071                trace_cdns3_ring(priv_ep);
1072                /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1073                writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1074
1075                priv_ep->prime_flag = false;
1076
1077                /*
1078                 * Controller version DEV_VER_V2 tdl calculation
1079                 * is based on TRB
1080                 */
1081
1082                if (priv_dev->dev_ver < DEV_VER_V2)
1083                        writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1084                               &priv_dev->regs->ep_cmd);
1085                else if (priv_dev->dev_ver > DEV_VER_V2)
1086                        writel(tdl, &priv_dev->regs->ep_tdl);
1087
1088                priv_ep->last_stream_id = priv_req->request.stream_id;
1089                writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1090                writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1091                       EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1092
1093                trace_cdns3_doorbell_epx(priv_ep->name,
1094                                         readl(&priv_dev->regs->ep_traddr));
1095        }
1096
1097        /* WORKAROUND for transition to L0 */
1098        __cdns3_gadget_wakeup(priv_dev);
1099
1100        return 0;
1101}
1102
1103/**
1104 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1105 * @priv_ep: endpoint object
1106 * @request: request object
1107 *
1108 * Returns zero on success or negative value on failure
1109 */
1110static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1111                                 struct usb_request *request)
1112{
1113        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1114        struct cdns3_request *priv_req;
1115        struct cdns3_trb *trb;
1116        struct cdns3_trb *link_trb = NULL;
1117        dma_addr_t trb_dma;
1118        u32 togle_pcs = 1;
1119        int sg_iter = 0;
1120        int num_trb;
1121        int address;
1122        u32 control;
1123        int pcs;
1124        u16 total_tdl = 0;
1125        struct scatterlist *s = NULL;
1126        bool sg_supported = !!(request->num_mapped_sgs);
1127
1128        if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1129                num_trb = priv_ep->interval;
1130        else
1131                num_trb = sg_supported ? request->num_mapped_sgs : 1;
1132
1133        if (num_trb > priv_ep->free_trbs) {
1134                priv_ep->flags |= EP_RING_FULL;
1135                return -ENOBUFS;
1136        }
1137
1138        priv_req = to_cdns3_request(request);
1139        address = priv_ep->endpoint.desc->bEndpointAddress;
1140
1141        priv_ep->flags |= EP_PENDING_REQUEST;
1142
1143        /* must allocate buffer aligned to 8 */
1144        if (priv_req->flags & REQUEST_UNALIGNED)
1145                trb_dma = priv_req->aligned_buf->dma;
1146        else
1147                trb_dma = request->dma;
1148
1149        trb = priv_ep->trb_pool + priv_ep->enqueue;
1150        priv_req->start_trb = priv_ep->enqueue;
1151        priv_req->trb = trb;
1152
1153        cdns3_select_ep(priv_ep->cdns3_dev, address);
1154
1155        /* prepare ring */
1156        if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
1157                int doorbell, dma_index;
1158                u32 ch_bit = 0;
1159
1160                doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1161                dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1162
1163                /* Driver can't update LINK TRB if it is current processed. */
1164                if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1165                        priv_ep->flags |= EP_DEFERRED_DRDY;
1166                        return -ENOBUFS;
1167                }
1168
1169                /*updating C bt in  Link TRB before starting DMA*/
1170                link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1171                /*
1172                 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1173                 * that DMA stuck at the LINK TRB.
1174                 * On the other hand, removing TRB_CHAIN for longer TRs for
1175                 * epXout cause that DMA stuck after handling LINK TRB.
1176                 * To eliminate this strange behavioral driver set TRB_CHAIN
1177                 * bit only for TR size > 2.
1178                 */
1179                if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1180                    TRBS_PER_SEGMENT > 2)
1181                        ch_bit = TRB_CHAIN;
1182
1183                link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1184                                    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1185        }
1186
1187        if (priv_dev->dev_ver <= DEV_VER_V2)
1188                togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1189
1190        if (sg_supported)
1191                s = request->sg;
1192
1193        /* set incorrect Cycle Bit for first trb*/
1194        control = priv_ep->pcs ? 0 : TRB_CYCLE;
1195        trb->length = 0;
1196        if (priv_dev->dev_ver >= DEV_VER_V2) {
1197                u16 td_size;
1198
1199                td_size = DIV_ROUND_UP(request->length,
1200                                       priv_ep->endpoint.maxpacket);
1201                if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1202                        trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1203                else
1204                        control |= TRB_TDL_HS_SIZE(td_size);
1205        }
1206
1207        do {
1208                u32 length;
1209
1210                /* fill TRB */
1211                control |= TRB_TYPE(TRB_NORMAL);
1212                if (sg_supported) {
1213                        trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1214                        length = sg_dma_len(s);
1215                } else {
1216                        trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1217                        length = request->length;
1218                }
1219
1220                if (priv_ep->flags & EP_TDLCHK_EN)
1221                        total_tdl += DIV_ROUND_UP(length,
1222                                               priv_ep->endpoint.maxpacket);
1223
1224                trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
1225                                        TRB_LEN(length));
1226                pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1227
1228                /*
1229                 * first trb should be prepared as last to avoid processing
1230                 *  transfer to early
1231                 */
1232                if (sg_iter != 0)
1233                        control |= pcs;
1234
1235                if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
1236                        control |= TRB_IOC | TRB_ISP;
1237                } else {
1238                        /* for last element in TD or in SG list */
1239                        if (sg_iter == (num_trb - 1) && sg_iter != 0)
1240                                control |= pcs | TRB_IOC | TRB_ISP;
1241                }
1242
1243                if (sg_iter)
1244                        trb->control = cpu_to_le32(control);
1245                else
1246                        priv_req->trb->control = cpu_to_le32(control);
1247
1248                if (sg_supported) {
1249                        trb->control |= cpu_to_le32(TRB_ISP);
1250                        /* Don't set chain bit for last TRB */
1251                        if (sg_iter < num_trb - 1)
1252                                trb->control |= cpu_to_le32(TRB_CHAIN);
1253
1254                        s = sg_next(s);
1255                }
1256
1257                control = 0;
1258                ++sg_iter;
1259                priv_req->end_trb = priv_ep->enqueue;
1260                cdns3_ep_inc_enq(priv_ep);
1261                trb = priv_ep->trb_pool + priv_ep->enqueue;
1262                trb->length = 0;
1263        } while (sg_iter < num_trb);
1264
1265        trb = priv_req->trb;
1266
1267        priv_req->flags |= REQUEST_PENDING;
1268        priv_req->num_of_trb = num_trb;
1269
1270        if (sg_iter == 1)
1271                trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
1272
1273        if (priv_dev->dev_ver < DEV_VER_V2 &&
1274            (priv_ep->flags & EP_TDLCHK_EN)) {
1275                u16 tdl = total_tdl;
1276                u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1277
1278                if (tdl > EP_CMD_TDL_MAX) {
1279                        tdl = EP_CMD_TDL_MAX;
1280                        priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1281                }
1282
1283                if (old_tdl < tdl) {
1284                        tdl -= old_tdl;
1285                        writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1286                               &priv_dev->regs->ep_cmd);
1287                }
1288        }
1289
1290        /*
1291         * Memory barrier - cycle bit must be set before other filds in trb.
1292         */
1293        wmb();
1294
1295        /* give the TD to the consumer*/
1296        if (togle_pcs)
1297                trb->control = trb->control ^ cpu_to_le32(1);
1298
1299        if (priv_dev->dev_ver <= DEV_VER_V2)
1300                cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1301
1302        if (num_trb > 1) {
1303                int i = 0;
1304
1305                while (i < num_trb) {
1306                        trace_cdns3_prepare_trb(priv_ep, trb + i);
1307                        if (trb + i == link_trb) {
1308                                trb = priv_ep->trb_pool;
1309                                num_trb = num_trb - i;
1310                                i = 0;
1311                        } else {
1312                                i++;
1313                        }
1314                }
1315        } else {
1316                trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1317        }
1318
1319        /*
1320         * Memory barrier - Cycle Bit must be set before trb->length  and
1321         * trb->buffer fields.
1322         */
1323        wmb();
1324
1325        /*
1326         * For DMULT mode we can set address to transfer ring only once after
1327         * enabling endpoint.
1328         */
1329        if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1330                /*
1331                 * Until SW is not ready to handle the OUT transfer the ISO OUT
1332                 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1333                 * EP_CFG_ENABLE must be set before updating ep_traddr.
1334                 */
1335                if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
1336                    !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1337                        priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1338                        cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1339                                               EP_CFG_ENABLE);
1340                }
1341
1342                writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1343                                        priv_req->start_trb * TRB_SIZE),
1344                                        &priv_dev->regs->ep_traddr);
1345
1346                priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1347        }
1348
1349        if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1350                trace_cdns3_ring(priv_ep);
1351                /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1352                writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1353                writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1354                trace_cdns3_doorbell_epx(priv_ep->name,
1355                                         readl(&priv_dev->regs->ep_traddr));
1356        }
1357
1358        /* WORKAROUND for transition to L0 */
1359        __cdns3_gadget_wakeup(priv_dev);
1360
1361        return 0;
1362}
1363
1364void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1365{
1366        struct cdns3_endpoint *priv_ep;
1367        struct usb_ep *ep;
1368
1369        if (priv_dev->hw_configured_flag)
1370                return;
1371
1372        writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1373
1374        cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1375                               USB_CONF_U1EN | USB_CONF_U2EN);
1376
1377        priv_dev->hw_configured_flag = 1;
1378
1379        list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1380                if (ep->enabled) {
1381                        priv_ep = ep_to_cdns3_ep(ep);
1382                        cdns3_start_all_request(priv_dev, priv_ep);
1383                }
1384        }
1385
1386        cdns3_allow_enable_l1(priv_dev, 1);
1387}
1388
1389/**
1390 * cdns3_trb_handled - check whether trb has been handled by DMA
1391 *
1392 * @priv_ep: extended endpoint object.
1393 * @priv_req: request object for checking
1394 *
1395 * Endpoint must be selected before invoking this function.
1396 *
1397 * Returns false if request has not been handled by DMA, else returns true.
1398 *
1399 * SR - start ring
1400 * ER -  end ring
1401 * DQ = priv_ep->dequeue - dequeue position
1402 * EQ = priv_ep->enqueue -  enqueue position
1403 * ST = priv_req->start_trb - index of first TRB in transfer ring
1404 * ET = priv_req->end_trb - index of last TRB in transfer ring
1405 * CI = current_index - index of processed TRB by DMA.
1406 *
1407 * As first step, we check if the TRB between the ST and ET.
1408 * Then, we check if cycle bit for index priv_ep->dequeue
1409 * is correct.
1410 *
1411 * some rules:
1412 * 1. priv_ep->dequeue never equals to current_index.
1413 * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1414 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1415 *    and priv_ep->free_trbs is zero.
1416 *    This case indicate that TR is full.
1417 *
1418 * At below two cases, the request have been handled.
1419 * Case 1 - priv_ep->dequeue < current_index
1420 *      SR ... EQ ... DQ ... CI ... ER
1421 *      SR ... DQ ... CI ... EQ ... ER
1422 *
1423 * Case 2 - priv_ep->dequeue > current_index
1424 * This situation takes place when CI go through the LINK TRB at the end of
1425 * transfer ring.
1426 *      SR ... CI ... EQ ... DQ ... ER
1427 */
1428static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1429                                  struct cdns3_request *priv_req)
1430{
1431        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1432        struct cdns3_trb *trb;
1433        int current_index = 0;
1434        int handled = 0;
1435        int doorbell;
1436
1437        current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1438        doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1439
1440        /* current trb doesn't belong to this request */
1441        if (priv_req->start_trb < priv_req->end_trb) {
1442                if (priv_ep->dequeue > priv_req->end_trb)
1443                        goto finish;
1444
1445                if (priv_ep->dequeue < priv_req->start_trb)
1446                        goto finish;
1447        }
1448
1449        if ((priv_req->start_trb > priv_req->end_trb) &&
1450                (priv_ep->dequeue > priv_req->end_trb) &&
1451                (priv_ep->dequeue < priv_req->start_trb))
1452                goto finish;
1453
1454        if ((priv_req->start_trb == priv_req->end_trb) &&
1455                (priv_ep->dequeue != priv_req->end_trb))
1456                goto finish;
1457
1458        trb = &priv_ep->trb_pool[priv_ep->dequeue];
1459
1460        if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1461                goto finish;
1462
1463        if (doorbell == 1 && current_index == priv_ep->dequeue)
1464                goto finish;
1465
1466        /* The corner case for TRBS_PER_SEGMENT equal 2). */
1467        if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1468                handled = 1;
1469                goto finish;
1470        }
1471
1472        if (priv_ep->enqueue == priv_ep->dequeue &&
1473            priv_ep->free_trbs == 0) {
1474                handled = 1;
1475        } else if (priv_ep->dequeue < current_index) {
1476                if ((current_index == (priv_ep->num_trbs - 1)) &&
1477                    !priv_ep->dequeue)
1478                        goto finish;
1479
1480                handled = 1;
1481        } else if (priv_ep->dequeue  > current_index) {
1482                        handled = 1;
1483        }
1484
1485finish:
1486        trace_cdns3_request_handled(priv_req, current_index, handled);
1487
1488        return handled;
1489}
1490
1491static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1492                                     struct cdns3_endpoint *priv_ep)
1493{
1494        struct cdns3_request *priv_req;
1495        struct usb_request *request;
1496        struct cdns3_trb *trb;
1497        bool request_handled = false;
1498        bool transfer_end = false;
1499
1500        while (!list_empty(&priv_ep->pending_req_list)) {
1501                request = cdns3_next_request(&priv_ep->pending_req_list);
1502                priv_req = to_cdns3_request(request);
1503
1504                trb = priv_ep->trb_pool + priv_ep->dequeue;
1505
1506                /* Request was dequeued and TRB was changed to TRB_LINK. */
1507                if (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1508                        trace_cdns3_complete_trb(priv_ep, trb);
1509                        cdns3_move_deq_to_next_trb(priv_req);
1510                }
1511
1512                if (!request->stream_id) {
1513                        /* Re-select endpoint. It could be changed by other CPU
1514                         * during handling usb_gadget_giveback_request.
1515                         */
1516                        cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1517
1518                        while (cdns3_trb_handled(priv_ep, priv_req)) {
1519                                priv_req->finished_trb++;
1520                                if (priv_req->finished_trb >= priv_req->num_of_trb)
1521                                        request_handled = true;
1522
1523                                trb = priv_ep->trb_pool + priv_ep->dequeue;
1524                                trace_cdns3_complete_trb(priv_ep, trb);
1525
1526                                if (!transfer_end)
1527                                        request->actual +=
1528                                                TRB_LEN(le32_to_cpu(trb->length));
1529
1530                                if (priv_req->num_of_trb > 1 &&
1531                                        le32_to_cpu(trb->control) & TRB_SMM)
1532                                        transfer_end = true;
1533
1534                                cdns3_ep_inc_deq(priv_ep);
1535                        }
1536
1537                        if (request_handled) {
1538                                cdns3_gadget_giveback(priv_ep, priv_req, 0);
1539                                request_handled = false;
1540                                transfer_end = false;
1541                        } else {
1542                                goto prepare_next_td;
1543                        }
1544
1545                        if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1546                            TRBS_PER_SEGMENT == 2)
1547                                break;
1548                } else {
1549                        /* Re-select endpoint. It could be changed by other CPU
1550                         * during handling usb_gadget_giveback_request.
1551                         */
1552                        cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1553
1554                        trb = priv_ep->trb_pool;
1555                        trace_cdns3_complete_trb(priv_ep, trb);
1556
1557                        if (trb != priv_req->trb)
1558                                dev_warn(priv_dev->dev,
1559                                         "request_trb=0x%p, queue_trb=0x%p\n",
1560                                         priv_req->trb, trb);
1561
1562                        request->actual += TRB_LEN(le32_to_cpu(trb->length));
1563
1564                        if (!request->num_sgs ||
1565                            (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1566                                priv_ep->stream_sg_idx = 0;
1567                                cdns3_gadget_giveback(priv_ep, priv_req, 0);
1568                        } else {
1569                                priv_ep->stream_sg_idx++;
1570                                cdns3_ep_run_stream_transfer(priv_ep, request);
1571                        }
1572                        break;
1573                }
1574        }
1575        priv_ep->flags &= ~EP_PENDING_REQUEST;
1576
1577prepare_next_td:
1578        if (!(priv_ep->flags & EP_STALLED) &&
1579            !(priv_ep->flags & EP_STALL_PENDING))
1580                cdns3_start_all_request(priv_dev, priv_ep);
1581}
1582
1583void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1584{
1585        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1586
1587        cdns3_wa1_restore_cycle_bit(priv_ep);
1588
1589        if (rearm) {
1590                trace_cdns3_ring(priv_ep);
1591
1592                /* Cycle Bit must be updated before arming DMA. */
1593                wmb();
1594                writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1595
1596                __cdns3_gadget_wakeup(priv_dev);
1597
1598                trace_cdns3_doorbell_epx(priv_ep->name,
1599                                         readl(&priv_dev->regs->ep_traddr));
1600        }
1601}
1602
1603static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1604{
1605        u16 tdl = priv_ep->pending_tdl;
1606        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1607
1608        if (tdl > EP_CMD_TDL_MAX) {
1609                tdl = EP_CMD_TDL_MAX;
1610                priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1611        } else {
1612                priv_ep->pending_tdl = 0;
1613        }
1614
1615        writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1616}
1617
1618/**
1619 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1620 * @priv_ep: endpoint object
1621 *
1622 * Returns 0
1623 */
1624static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1625{
1626        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1627        u32 ep_sts_reg;
1628        struct usb_request *deferred_request;
1629        struct usb_request *pending_request;
1630        u32 tdl = 0;
1631
1632        cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1633
1634        trace_cdns3_epx_irq(priv_dev, priv_ep);
1635
1636        ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1637        writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1638
1639        if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1640                bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1641
1642                tdl = cdns3_get_tdl(priv_dev);
1643
1644                /*
1645                 * Continue the previous transfer:
1646                 * There is some racing between ERDY and PRIME. The device send
1647                 * ERDY and almost in the same time Host send PRIME. It cause
1648                 * that host ignore the ERDY packet and driver has to send it
1649                 * again.
1650                 */
1651                if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1652                    EP_STS_HOSTPP(ep_sts_reg))) {
1653                        writel(EP_CMD_ERDY |
1654                               EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1655                               &priv_dev->regs->ep_cmd);
1656                        ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1657                } else {
1658                        priv_ep->prime_flag = true;
1659
1660                        pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1661                        deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1662
1663                        if (deferred_request && !pending_request) {
1664                                cdns3_start_all_request(priv_dev, priv_ep);
1665                        }
1666                }
1667        }
1668
1669        if (ep_sts_reg & EP_STS_TRBERR) {
1670                if (priv_ep->flags & EP_STALL_PENDING &&
1671                    !(ep_sts_reg & EP_STS_DESCMIS &&
1672                    priv_dev->dev_ver < DEV_VER_V2)) {
1673                        cdns3_ep_stall_flush(priv_ep);
1674                }
1675
1676                /*
1677                 * For isochronous transfer driver completes request on
1678                 * IOC or on TRBERR. IOC appears only when device receive
1679                 * OUT data packet. If host disable stream or lost some packet
1680                 * then the only way to finish all queued transfer is to do it
1681                 * on TRBERR event.
1682                 */
1683                if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1684                    !priv_ep->wa1_set) {
1685                        if (!priv_ep->dir) {
1686                                u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1687
1688                                ep_cfg &= ~EP_CFG_ENABLE;
1689                                writel(ep_cfg, &priv_dev->regs->ep_cfg);
1690                                priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1691                        }
1692                        cdns3_transfer_completed(priv_dev, priv_ep);
1693                } else if (!(priv_ep->flags & EP_STALLED) &&
1694                          !(priv_ep->flags & EP_STALL_PENDING)) {
1695                        if (priv_ep->flags & EP_DEFERRED_DRDY) {
1696                                priv_ep->flags &= ~EP_DEFERRED_DRDY;
1697                                cdns3_start_all_request(priv_dev, priv_ep);
1698                        } else {
1699                                cdns3_rearm_transfer(priv_ep,
1700                                                     priv_ep->wa1_set);
1701                        }
1702                }
1703        }
1704
1705        if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1706            (ep_sts_reg & EP_STS_IOT)) {
1707                if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1708                        if (ep_sts_reg & EP_STS_ISP)
1709                                priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1710                        else
1711                                priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1712                }
1713
1714                if (!priv_ep->use_streams) {
1715                        if ((ep_sts_reg & EP_STS_IOC) ||
1716                            (ep_sts_reg & EP_STS_ISP)) {
1717                                cdns3_transfer_completed(priv_dev, priv_ep);
1718                        } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1719                                   priv_ep->pending_tdl) {
1720                                /* handle IOT with pending tdl */
1721                                cdns3_reprogram_tdl(priv_ep);
1722                        }
1723                } else if (priv_ep->dir == USB_DIR_OUT) {
1724                        priv_ep->ep_sts_pending |= ep_sts_reg;
1725                } else if (ep_sts_reg & EP_STS_IOT) {
1726                        cdns3_transfer_completed(priv_dev, priv_ep);
1727                }
1728        }
1729
1730        /*
1731         * MD_EXIT interrupt sets when stream capable endpoint exits
1732         * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1733         */
1734        if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1735            (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1736                priv_ep->ep_sts_pending = 0;
1737                cdns3_transfer_completed(priv_dev, priv_ep);
1738        }
1739
1740        /*
1741         * WA2: this condition should only be meet when
1742         * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1743         * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1744         * In other cases this interrupt will be disabled.
1745         */
1746        if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1747            !(priv_ep->flags & EP_STALLED))
1748                cdns3_wa2_descmissing_packet(priv_ep);
1749
1750        return 0;
1751}
1752
1753static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1754{
1755        if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1756                priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1757}
1758
1759/**
1760 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1761 * @priv_dev: extended gadget object
1762 * @usb_ists: bitmap representation of device's reported interrupts
1763 * (usb_ists register value)
1764 */
1765static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1766                                              u32 usb_ists)
1767__must_hold(&priv_dev->lock)
1768{
1769        int speed = 0;
1770
1771        trace_cdns3_usb_irq(priv_dev, usb_ists);
1772        if (usb_ists & USB_ISTS_L1ENTI) {
1773                /*
1774                 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1775                 * from L1. To fix it, if any DMA transfer is pending driver
1776                 * must starts driving resume signal immediately.
1777                 */
1778                if (readl(&priv_dev->regs->drbl))
1779                        __cdns3_gadget_wakeup(priv_dev);
1780        }
1781
1782        /* Connection detected */
1783        if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1784                speed = cdns3_get_speed(priv_dev);
1785                priv_dev->gadget.speed = speed;
1786                usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1787                cdns3_ep0_config(priv_dev);
1788        }
1789
1790        /* Disconnection detected */
1791        if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1792                spin_unlock(&priv_dev->lock);
1793                cdns3_disconnect_gadget(priv_dev);
1794                spin_lock(&priv_dev->lock);
1795                priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1796                usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1797                cdns3_hw_reset_eps_config(priv_dev);
1798        }
1799
1800        if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1801                if (priv_dev->gadget_driver &&
1802                    priv_dev->gadget_driver->suspend) {
1803                        spin_unlock(&priv_dev->lock);
1804                        priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1805                        spin_lock(&priv_dev->lock);
1806                }
1807        }
1808
1809        if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1810                if (priv_dev->gadget_driver &&
1811                    priv_dev->gadget_driver->resume) {
1812                        spin_unlock(&priv_dev->lock);
1813                        priv_dev->gadget_driver->resume(&priv_dev->gadget);
1814                        spin_lock(&priv_dev->lock);
1815                }
1816        }
1817
1818        /* reset*/
1819        if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1820                if (priv_dev->gadget_driver) {
1821                        spin_unlock(&priv_dev->lock);
1822                        usb_gadget_udc_reset(&priv_dev->gadget,
1823                                             priv_dev->gadget_driver);
1824                        spin_lock(&priv_dev->lock);
1825
1826                        /*read again to check the actual speed*/
1827                        speed = cdns3_get_speed(priv_dev);
1828                        priv_dev->gadget.speed = speed;
1829                        cdns3_hw_reset_eps_config(priv_dev);
1830                        cdns3_ep0_config(priv_dev);
1831                }
1832        }
1833}
1834
1835/**
1836 * cdns3_device_irq_handler - interrupt handler for device part of controller
1837 *
1838 * @irq: irq number for cdns3 core device
1839 * @data: structure of cdns3
1840 *
1841 * Returns IRQ_HANDLED or IRQ_NONE
1842 */
1843static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1844{
1845        struct cdns3_device *priv_dev = data;
1846        struct cdns *cdns = dev_get_drvdata(priv_dev->dev);
1847        irqreturn_t ret = IRQ_NONE;
1848        u32 reg;
1849
1850        if (cdns->in_lpm)
1851                return ret;
1852
1853        /* check USB device interrupt */
1854        reg = readl(&priv_dev->regs->usb_ists);
1855        if (reg) {
1856                /* After masking interrupts the new interrupts won't be
1857                 * reported in usb_ists/ep_ists. In order to not lose some
1858                 * of them driver disables only detected interrupts.
1859                 * They will be enabled ASAP after clearing source of
1860                 * interrupt. This an unusual behavior only applies to
1861                 * usb_ists register.
1862                 */
1863                reg = ~reg & readl(&priv_dev->regs->usb_ien);
1864                /* mask deferred interrupt. */
1865                writel(reg, &priv_dev->regs->usb_ien);
1866                ret = IRQ_WAKE_THREAD;
1867        }
1868
1869        /* check endpoint interrupt */
1870        reg = readl(&priv_dev->regs->ep_ists);
1871        if (reg) {
1872                writel(0, &priv_dev->regs->ep_ien);
1873                ret = IRQ_WAKE_THREAD;
1874        }
1875
1876        return ret;
1877}
1878
1879/**
1880 * cdns3_device_thread_irq_handler - interrupt handler for device part
1881 * of controller
1882 *
1883 * @irq: irq number for cdns3 core device
1884 * @data: structure of cdns3
1885 *
1886 * Returns IRQ_HANDLED or IRQ_NONE
1887 */
1888static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1889{
1890        struct cdns3_device *priv_dev = data;
1891        irqreturn_t ret = IRQ_NONE;
1892        unsigned long flags;
1893        unsigned int bit;
1894        unsigned long reg;
1895
1896        spin_lock_irqsave(&priv_dev->lock, flags);
1897
1898        reg = readl(&priv_dev->regs->usb_ists);
1899        if (reg) {
1900                writel(reg, &priv_dev->regs->usb_ists);
1901                writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1902                cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1903                ret = IRQ_HANDLED;
1904        }
1905
1906        reg = readl(&priv_dev->regs->ep_ists);
1907
1908        /* handle default endpoint OUT */
1909        if (reg & EP_ISTS_EP_OUT0) {
1910                cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1911                ret = IRQ_HANDLED;
1912        }
1913
1914        /* handle default endpoint IN */
1915        if (reg & EP_ISTS_EP_IN0) {
1916                cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1917                ret = IRQ_HANDLED;
1918        }
1919
1920        /* check if interrupt from non default endpoint, if no exit */
1921        reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1922        if (!reg)
1923                goto irqend;
1924
1925        for_each_set_bit(bit, &reg,
1926                         sizeof(u32) * BITS_PER_BYTE) {
1927                cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1928                ret = IRQ_HANDLED;
1929        }
1930
1931        if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1932                cdns3_wa2_check_outq_status(priv_dev);
1933
1934irqend:
1935        writel(~0, &priv_dev->regs->ep_ien);
1936        spin_unlock_irqrestore(&priv_dev->lock, flags);
1937
1938        return ret;
1939}
1940
1941/**
1942 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1943 *
1944 * The real reservation will occur during write to EP_CFG register,
1945 * this function is used to check if the 'size' reservation is allowed.
1946 *
1947 * @priv_dev: extended gadget object
1948 * @size: the size (KB) for EP would like to allocate
1949 * @is_in: endpoint direction
1950 *
1951 * Return 0 if the required size can met or negative value on failure
1952 */
1953static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1954                                          int size, int is_in)
1955{
1956        int remained;
1957
1958        /* 2KB are reserved for EP0*/
1959        remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1960
1961        if (is_in) {
1962                if (remained < size)
1963                        return -EPERM;
1964
1965                priv_dev->onchip_used_size += size;
1966        } else {
1967                int required;
1968
1969                /**
1970                 *  ALL OUT EPs are shared the same chunk onchip memory, so
1971                 * driver checks if it already has assigned enough buffers
1972                 */
1973                if (priv_dev->out_mem_is_allocated >= size)
1974                        return 0;
1975
1976                required = size - priv_dev->out_mem_is_allocated;
1977
1978                if (required > remained)
1979                        return -EPERM;
1980
1981                priv_dev->out_mem_is_allocated += required;
1982                priv_dev->onchip_used_size += required;
1983        }
1984
1985        return 0;
1986}
1987
1988static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1989                                  struct cdns3_endpoint *priv_ep)
1990{
1991        struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1992
1993        /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1994        if (priv_dev->dev_ver <= DEV_VER_V2)
1995                writel(USB_CONF_DMULT, &regs->usb_conf);
1996
1997        if (priv_dev->dev_ver == DEV_VER_V2)
1998                writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1999
2000        if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2001                u32 mask;
2002
2003                if (priv_ep->dir)
2004                        mask = BIT(priv_ep->num + 16);
2005                else
2006                        mask = BIT(priv_ep->num);
2007
2008                if (priv_ep->type != USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
2009                        cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2010                        cdns3_set_register_bit(&regs->tdl_beh, mask);
2011                        cdns3_set_register_bit(&regs->tdl_beh2, mask);
2012                        cdns3_set_register_bit(&regs->dma_adv_td, mask);
2013                }
2014
2015                if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2016                        cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2017
2018                cdns3_set_register_bit(&regs->dtrans, mask);
2019        }
2020}
2021
2022/**
2023 * cdns3_ep_config - Configure hardware endpoint
2024 * @priv_ep: extended endpoint object
2025 * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2026 */
2027int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2028{
2029        bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2030        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2031        u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2032        u32 max_packet_size = 0;
2033        u8 maxburst = 0;
2034        u32 ep_cfg = 0;
2035        u8 buffering;
2036        u8 mult = 0;
2037        int ret;
2038
2039        buffering = CDNS3_EP_BUF_SIZE - 1;
2040
2041        cdns3_configure_dmult(priv_dev, priv_ep);
2042
2043        switch (priv_ep->type) {
2044        case USB_ENDPOINT_XFER_INT:
2045                ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2046
2047                if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2048                        ep_cfg |= EP_CFG_TDL_CHK;
2049                break;
2050        case USB_ENDPOINT_XFER_BULK:
2051                ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2052
2053                if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2054                        ep_cfg |= EP_CFG_TDL_CHK;
2055                break;
2056        default:
2057                ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2058                mult = CDNS3_EP_ISO_HS_MULT - 1;
2059                buffering = mult + 1;
2060        }
2061
2062        switch (priv_dev->gadget.speed) {
2063        case USB_SPEED_FULL:
2064                max_packet_size = is_iso_ep ? 1023 : 64;
2065                break;
2066        case USB_SPEED_HIGH:
2067                max_packet_size = is_iso_ep ? 1024 : 512;
2068                break;
2069        case USB_SPEED_SUPER:
2070                /* It's limitation that driver assumes in driver. */
2071                mult = 0;
2072                max_packet_size = 1024;
2073                if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2074                        maxburst = CDNS3_EP_ISO_SS_BURST - 1;
2075                        buffering = (mult + 1) *
2076                                    (maxburst + 1);
2077
2078                        if (priv_ep->interval > 1)
2079                                buffering++;
2080                } else {
2081                        maxburst = CDNS3_EP_BUF_SIZE - 1;
2082                }
2083                break;
2084        default:
2085                /* all other speed are not supported */
2086                return -EINVAL;
2087        }
2088
2089        if (max_packet_size == 1024)
2090                priv_ep->trb_burst_size = 128;
2091        else if (max_packet_size >= 512)
2092                priv_ep->trb_burst_size = 64;
2093        else
2094                priv_ep->trb_burst_size = 16;
2095
2096        /* onchip buffer is only allocated before configuration */
2097        if (!priv_dev->hw_configured_flag) {
2098                ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2099                                                     !!priv_ep->dir);
2100                if (ret) {
2101                        dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2102                        return ret;
2103                }
2104        }
2105
2106        if (enable)
2107                ep_cfg |= EP_CFG_ENABLE;
2108
2109        if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2110                if (priv_dev->dev_ver >= DEV_VER_V3) {
2111                        u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2112
2113                        /*
2114                         * Stream capable endpoints are handled by using ep_tdl
2115                         * register. Other endpoints use TDL from TRB feature.
2116                         */
2117                        cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2118                                                 mask);
2119                }
2120
2121                /*  Enable Stream Bit TDL chk and SID chk */
2122                ep_cfg |=  EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2123        }
2124
2125        ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2126                  EP_CFG_MULT(mult) |
2127                  EP_CFG_BUFFERING(buffering) |
2128                  EP_CFG_MAXBURST(maxburst);
2129
2130        cdns3_select_ep(priv_dev, bEndpointAddress);
2131        writel(ep_cfg, &priv_dev->regs->ep_cfg);
2132        priv_ep->flags |= EP_CONFIGURED;
2133
2134        dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2135                priv_ep->name, ep_cfg);
2136
2137        return 0;
2138}
2139
2140/* Find correct direction for HW endpoint according to description */
2141static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2142                                   struct cdns3_endpoint *priv_ep)
2143{
2144        return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2145               (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2146}
2147
2148static struct
2149cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2150                                        struct usb_endpoint_descriptor *desc)
2151{
2152        struct usb_ep *ep;
2153        struct cdns3_endpoint *priv_ep;
2154
2155        list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2156                unsigned long num;
2157                int ret;
2158                /* ep name pattern likes epXin or epXout */
2159                char c[2] = {ep->name[2], '\0'};
2160
2161                ret = kstrtoul(c, 10, &num);
2162                if (ret)
2163                        return ERR_PTR(ret);
2164
2165                priv_ep = ep_to_cdns3_ep(ep);
2166                if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2167                        if (!(priv_ep->flags & EP_CLAIMED)) {
2168                                priv_ep->num  = num;
2169                                return priv_ep;
2170                        }
2171                }
2172        }
2173
2174        return ERR_PTR(-ENOENT);
2175}
2176
2177/*
2178 *  Cadence IP has one limitation that all endpoints must be configured
2179 * (Type & MaxPacketSize) before setting configuration through hardware
2180 * register, it means we can't change endpoints configuration after
2181 * set_configuration.
2182 *
2183 * This function set EP_CLAIMED flag which is added when the gadget driver
2184 * uses usb_ep_autoconfig to configure specific endpoint;
2185 * When the udc driver receives set_configurion request,
2186 * it goes through all claimed endpoints, and configure all endpoints
2187 * accordingly.
2188 *
2189 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2190 * ep_cfg register which can be changed after set_configuration, and do
2191 * some software operation accordingly.
2192 */
2193static struct
2194usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2195                              struct usb_endpoint_descriptor *desc,
2196                              struct usb_ss_ep_comp_descriptor *comp_desc)
2197{
2198        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2199        struct cdns3_endpoint *priv_ep;
2200        unsigned long flags;
2201
2202        priv_ep = cdns3_find_available_ep(priv_dev, desc);
2203        if (IS_ERR(priv_ep)) {
2204                dev_err(priv_dev->dev, "no available ep\n");
2205                return NULL;
2206        }
2207
2208        dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2209
2210        spin_lock_irqsave(&priv_dev->lock, flags);
2211        priv_ep->endpoint.desc = desc;
2212        priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2213        priv_ep->type = usb_endpoint_type(desc);
2214        priv_ep->flags |= EP_CLAIMED;
2215        priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2216
2217        spin_unlock_irqrestore(&priv_dev->lock, flags);
2218        return &priv_ep->endpoint;
2219}
2220
2221/**
2222 * cdns3_gadget_ep_alloc_request - Allocates request
2223 * @ep: endpoint object associated with request
2224 * @gfp_flags: gfp flags
2225 *
2226 * Returns allocated request address, NULL on allocation error
2227 */
2228struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2229                                                  gfp_t gfp_flags)
2230{
2231        struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2232        struct cdns3_request *priv_req;
2233
2234        priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2235        if (!priv_req)
2236                return NULL;
2237
2238        priv_req->priv_ep = priv_ep;
2239
2240        trace_cdns3_alloc_request(priv_req);
2241        return &priv_req->request;
2242}
2243
2244/**
2245 * cdns3_gadget_ep_free_request - Free memory occupied by request
2246 * @ep: endpoint object associated with request
2247 * @request: request to free memory
2248 */
2249void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2250                                  struct usb_request *request)
2251{
2252        struct cdns3_request *priv_req = to_cdns3_request(request);
2253
2254        if (priv_req->aligned_buf)
2255                priv_req->aligned_buf->in_use = 0;
2256
2257        trace_cdns3_free_request(priv_req);
2258        kfree(priv_req);
2259}
2260
2261/**
2262 * cdns3_gadget_ep_enable - Enable endpoint
2263 * @ep: endpoint object
2264 * @desc: endpoint descriptor
2265 *
2266 * Returns 0 on success, error code elsewhere
2267 */
2268static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2269                                  const struct usb_endpoint_descriptor *desc)
2270{
2271        struct cdns3_endpoint *priv_ep;
2272        struct cdns3_device *priv_dev;
2273        const struct usb_ss_ep_comp_descriptor *comp_desc;
2274        u32 reg = EP_STS_EN_TRBERREN;
2275        u32 bEndpointAddress;
2276        unsigned long flags;
2277        int enable = 1;
2278        int ret = 0;
2279        int val;
2280
2281        priv_ep = ep_to_cdns3_ep(ep);
2282        priv_dev = priv_ep->cdns3_dev;
2283        comp_desc = priv_ep->endpoint.comp_desc;
2284
2285        if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2286                dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2287                return -EINVAL;
2288        }
2289
2290        if (!desc->wMaxPacketSize) {
2291                dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2292                return -EINVAL;
2293        }
2294
2295        if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2296                          "%s is already enabled\n", priv_ep->name))
2297                return 0;
2298
2299        spin_lock_irqsave(&priv_dev->lock, flags);
2300
2301        priv_ep->endpoint.desc = desc;
2302        priv_ep->type = usb_endpoint_type(desc);
2303        priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2304
2305        if (priv_ep->interval > ISO_MAX_INTERVAL &&
2306            priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2307                dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2308                        ISO_MAX_INTERVAL);
2309
2310                ret =  -EINVAL;
2311                goto exit;
2312        }
2313
2314        bEndpointAddress = priv_ep->num | priv_ep->dir;
2315        cdns3_select_ep(priv_dev, bEndpointAddress);
2316
2317        /*
2318         * For some versions of controller at some point during ISO OUT traffic
2319         * DMA reads Transfer Ring for the EP which has never got doorbell.
2320         * This issue was detected only on simulation, but to avoid this issue
2321         * driver add protection against it. To fix it driver enable ISO OUT
2322         * endpoint before setting DRBL. This special treatment of ISO OUT
2323         * endpoints are recommended by controller specification.
2324         */
2325        if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
2326                enable = 0;
2327
2328        if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2329                /*
2330                 * Enable stream support (SS mode) related interrupts
2331                 * in EP_STS_EN Register
2332                 */
2333                if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2334                        reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2335                                EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2336                                EP_STS_EN_STREAMREN;
2337                        priv_ep->use_streams = true;
2338                        ret = cdns3_ep_config(priv_ep, enable);
2339                        priv_dev->using_streams |= true;
2340                }
2341        } else {
2342                ret = cdns3_ep_config(priv_ep, enable);
2343        }
2344
2345        if (ret)
2346                goto exit;
2347
2348        ret = cdns3_allocate_trb_pool(priv_ep);
2349        if (ret)
2350                goto exit;
2351
2352        bEndpointAddress = priv_ep->num | priv_ep->dir;
2353        cdns3_select_ep(priv_dev, bEndpointAddress);
2354
2355        trace_cdns3_gadget_ep_enable(priv_ep);
2356
2357        writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2358
2359        ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2360                                        !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2361                                        1, 1000);
2362
2363        if (unlikely(ret)) {
2364                cdns3_free_trb_pool(priv_ep);
2365                ret =  -EINVAL;
2366                goto exit;
2367        }
2368
2369        /* enable interrupt for selected endpoint */
2370        cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2371                               BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2372
2373        if (priv_dev->dev_ver < DEV_VER_V2)
2374                cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2375
2376        writel(reg, &priv_dev->regs->ep_sts_en);
2377
2378        ep->desc = desc;
2379        priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2380                            EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2381        priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2382        priv_ep->wa1_set = 0;
2383        priv_ep->enqueue = 0;
2384        priv_ep->dequeue = 0;
2385        reg = readl(&priv_dev->regs->ep_sts);
2386        priv_ep->pcs = !!EP_STS_CCS(reg);
2387        priv_ep->ccs = !!EP_STS_CCS(reg);
2388        /* one TRB is reserved for link TRB used in DMULT mode*/
2389        priv_ep->free_trbs = priv_ep->num_trbs - 1;
2390exit:
2391        spin_unlock_irqrestore(&priv_dev->lock, flags);
2392
2393        return ret;
2394}
2395
2396/**
2397 * cdns3_gadget_ep_disable - Disable endpoint
2398 * @ep: endpoint object
2399 *
2400 * Returns 0 on success, error code elsewhere
2401 */
2402static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2403{
2404        struct cdns3_endpoint *priv_ep;
2405        struct cdns3_request *priv_req;
2406        struct cdns3_device *priv_dev;
2407        struct usb_request *request;
2408        unsigned long flags;
2409        int ret = 0;
2410        u32 ep_cfg;
2411        int val;
2412
2413        if (!ep) {
2414                pr_err("usbss: invalid parameters\n");
2415                return -EINVAL;
2416        }
2417
2418        priv_ep = ep_to_cdns3_ep(ep);
2419        priv_dev = priv_ep->cdns3_dev;
2420
2421        if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2422                          "%s is already disabled\n", priv_ep->name))
2423                return 0;
2424
2425        spin_lock_irqsave(&priv_dev->lock, flags);
2426
2427        trace_cdns3_gadget_ep_disable(priv_ep);
2428
2429        cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2430
2431        ep_cfg = readl(&priv_dev->regs->ep_cfg);
2432        ep_cfg &= ~EP_CFG_ENABLE;
2433        writel(ep_cfg, &priv_dev->regs->ep_cfg);
2434
2435        /**
2436         * Driver needs some time before resetting endpoint.
2437         * It need waits for clearing DBUSY bit or for timeout expired.
2438         * 10us is enough time for controller to stop transfer.
2439         */
2440        readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2441                                  !(val & EP_STS_DBUSY), 1, 10);
2442        writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2443
2444        readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2445                                  !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2446                                  1, 1000);
2447        if (unlikely(ret))
2448                dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2449                        priv_ep->name);
2450
2451        while (!list_empty(&priv_ep->pending_req_list)) {
2452                request = cdns3_next_request(&priv_ep->pending_req_list);
2453
2454                cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2455                                      -ESHUTDOWN);
2456        }
2457
2458        while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2459                priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2460
2461                kfree(priv_req->request.buf);
2462                cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2463                                             &priv_req->request);
2464                list_del_init(&priv_req->list);
2465                --priv_ep->wa2_counter;
2466        }
2467
2468        while (!list_empty(&priv_ep->deferred_req_list)) {
2469                request = cdns3_next_request(&priv_ep->deferred_req_list);
2470
2471                cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2472                                      -ESHUTDOWN);
2473        }
2474
2475        priv_ep->descmis_req = NULL;
2476
2477        ep->desc = NULL;
2478        priv_ep->flags &= ~EP_ENABLED;
2479        priv_ep->use_streams = false;
2480
2481        spin_unlock_irqrestore(&priv_dev->lock, flags);
2482
2483        return ret;
2484}
2485
2486/**
2487 * __cdns3_gadget_ep_queue - Transfer data on endpoint
2488 * @ep: endpoint object
2489 * @request: request object
2490 * @gfp_flags: gfp flags
2491 *
2492 * Returns 0 on success, error code elsewhere
2493 */
2494static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2495                                   struct usb_request *request,
2496                                   gfp_t gfp_flags)
2497{
2498        struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2499        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2500        struct cdns3_request *priv_req;
2501        int ret = 0;
2502
2503        request->actual = 0;
2504        request->status = -EINPROGRESS;
2505        priv_req = to_cdns3_request(request);
2506        trace_cdns3_ep_queue(priv_req);
2507
2508        if (priv_dev->dev_ver < DEV_VER_V2) {
2509                ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2510                                                priv_req);
2511
2512                if (ret == EINPROGRESS)
2513                        return 0;
2514        }
2515
2516        ret = cdns3_prepare_aligned_request_buf(priv_req);
2517        if (ret < 0)
2518                return ret;
2519
2520        ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2521                                            usb_endpoint_dir_in(ep->desc));
2522        if (ret)
2523                return ret;
2524
2525        list_add_tail(&request->list, &priv_ep->deferred_req_list);
2526
2527        /*
2528         * For stream capable endpoint if prime irq flag is set then only start
2529         * request.
2530         * If hardware endpoint configuration has not been set yet then
2531         * just queue request in deferred list. Transfer will be started in
2532         * cdns3_set_hw_configuration.
2533         */
2534        if (!request->stream_id) {
2535                if (priv_dev->hw_configured_flag &&
2536                    !(priv_ep->flags & EP_STALLED) &&
2537                    !(priv_ep->flags & EP_STALL_PENDING))
2538                        cdns3_start_all_request(priv_dev, priv_ep);
2539        } else {
2540                if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2541                        cdns3_start_all_request(priv_dev, priv_ep);
2542        }
2543
2544        return 0;
2545}
2546
2547static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2548                                 gfp_t gfp_flags)
2549{
2550        struct usb_request *zlp_request;
2551        struct cdns3_endpoint *priv_ep;
2552        struct cdns3_device *priv_dev;
2553        unsigned long flags;
2554        int ret;
2555
2556        if (!request || !ep)
2557                return -EINVAL;
2558
2559        priv_ep = ep_to_cdns3_ep(ep);
2560        priv_dev = priv_ep->cdns3_dev;
2561
2562        spin_lock_irqsave(&priv_dev->lock, flags);
2563
2564        ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2565
2566        if (ret == 0 && request->zero && request->length &&
2567            (request->length % ep->maxpacket == 0)) {
2568                struct cdns3_request *priv_req;
2569
2570                zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2571                zlp_request->buf = priv_dev->zlp_buf;
2572                zlp_request->length = 0;
2573
2574                priv_req = to_cdns3_request(zlp_request);
2575                priv_req->flags |= REQUEST_ZLP;
2576
2577                dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2578                        priv_ep->name);
2579                ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2580        }
2581
2582        spin_unlock_irqrestore(&priv_dev->lock, flags);
2583        return ret;
2584}
2585
2586/**
2587 * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2588 * @ep: endpoint object associated with request
2589 * @request: request object
2590 *
2591 * Returns 0 on success, error code elsewhere
2592 */
2593int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2594                            struct usb_request *request)
2595{
2596        struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2597        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2598        struct usb_request *req, *req_temp;
2599        struct cdns3_request *priv_req;
2600        struct cdns3_trb *link_trb;
2601        u8 req_on_hw_ring = 0;
2602        unsigned long flags;
2603        int ret = 0;
2604
2605        if (!ep || !request || !ep->desc)
2606                return -EINVAL;
2607
2608        spin_lock_irqsave(&priv_dev->lock, flags);
2609
2610        priv_req = to_cdns3_request(request);
2611
2612        trace_cdns3_ep_dequeue(priv_req);
2613
2614        cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2615
2616        list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2617                                 list) {
2618                if (request == req) {
2619                        req_on_hw_ring = 1;
2620                        goto found;
2621                }
2622        }
2623
2624        list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2625                                 list) {
2626                if (request == req)
2627                        goto found;
2628        }
2629
2630        goto not_found;
2631
2632found:
2633        link_trb = priv_req->trb;
2634
2635        /* Update ring only if removed request is on pending_req_list list */
2636        if (req_on_hw_ring && link_trb) {
2637                link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2638                        ((priv_req->end_trb + 1) * TRB_SIZE)));
2639                link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2640                                    TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2641
2642                if (priv_ep->wa1_trb == priv_req->trb)
2643                        cdns3_wa1_restore_cycle_bit(priv_ep);
2644        }
2645
2646        cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2647
2648not_found:
2649        spin_unlock_irqrestore(&priv_dev->lock, flags);
2650        return ret;
2651}
2652
2653/**
2654 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2655 * Should be called after acquiring spin_lock and selecting ep
2656 * @priv_ep: endpoint object to set stall on.
2657 */
2658void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2659{
2660        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2661
2662        trace_cdns3_halt(priv_ep, 1, 0);
2663
2664        if (!(priv_ep->flags & EP_STALLED)) {
2665                u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2666
2667                if (!(ep_sts_reg & EP_STS_DBUSY))
2668                        cdns3_ep_stall_flush(priv_ep);
2669                else
2670                        priv_ep->flags |= EP_STALL_PENDING;
2671        }
2672}
2673
2674/**
2675 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2676 * Should be called after acquiring spin_lock and selecting ep
2677 * @priv_ep: endpoint object to clear stall on
2678 */
2679int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2680{
2681        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2682        struct usb_request *request;
2683        struct cdns3_request *priv_req;
2684        struct cdns3_trb *trb = NULL;
2685        int ret;
2686        int val;
2687
2688        trace_cdns3_halt(priv_ep, 0, 0);
2689
2690        request = cdns3_next_request(&priv_ep->pending_req_list);
2691        if (request) {
2692                priv_req = to_cdns3_request(request);
2693                trb = priv_req->trb;
2694                if (trb)
2695                        trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2696        }
2697
2698        writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2699
2700        /* wait for EPRST cleared */
2701        ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2702                                        !(val & EP_CMD_EPRST), 1, 100);
2703        if (ret)
2704                return -EINVAL;
2705
2706        priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2707
2708        if (request) {
2709                if (trb)
2710                        trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2711
2712                cdns3_rearm_transfer(priv_ep, 1);
2713        }
2714
2715        cdns3_start_all_request(priv_dev, priv_ep);
2716        return ret;
2717}
2718
2719/**
2720 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2721 * @ep: endpoint object to set/clear stall on
2722 * @value: 1 for set stall, 0 for clear stall
2723 *
2724 * Returns 0 on success, error code elsewhere
2725 */
2726int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2727{
2728        struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2729        struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2730        unsigned long flags;
2731        int ret = 0;
2732
2733        if (!(priv_ep->flags & EP_ENABLED))
2734                return -EPERM;
2735
2736        spin_lock_irqsave(&priv_dev->lock, flags);
2737
2738        cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2739
2740        if (!value) {
2741                priv_ep->flags &= ~EP_WEDGE;
2742                ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2743        } else {
2744                __cdns3_gadget_ep_set_halt(priv_ep);
2745        }
2746
2747        spin_unlock_irqrestore(&priv_dev->lock, flags);
2748
2749        return ret;
2750}
2751
2752extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2753
2754static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2755        .enable = cdns3_gadget_ep_enable,
2756        .disable = cdns3_gadget_ep_disable,
2757        .alloc_request = cdns3_gadget_ep_alloc_request,
2758        .free_request = cdns3_gadget_ep_free_request,
2759        .queue = cdns3_gadget_ep_queue,
2760        .dequeue = cdns3_gadget_ep_dequeue,
2761        .set_halt = cdns3_gadget_ep_set_halt,
2762        .set_wedge = cdns3_gadget_ep_set_wedge,
2763};
2764
2765/**
2766 * cdns3_gadget_get_frame - Returns number of actual ITP frame
2767 * @gadget: gadget object
2768 *
2769 * Returns number of actual ITP frame
2770 */
2771static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2772{
2773        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2774
2775        return readl(&priv_dev->regs->usb_itpn);
2776}
2777
2778int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2779{
2780        enum usb_device_speed speed;
2781
2782        speed = cdns3_get_speed(priv_dev);
2783
2784        if (speed >= USB_SPEED_SUPER)
2785                return 0;
2786
2787        /* Start driving resume signaling to indicate remote wakeup. */
2788        writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2789
2790        return 0;
2791}
2792
2793static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2794{
2795        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2796        unsigned long flags;
2797        int ret = 0;
2798
2799        spin_lock_irqsave(&priv_dev->lock, flags);
2800        ret = __cdns3_gadget_wakeup(priv_dev);
2801        spin_unlock_irqrestore(&priv_dev->lock, flags);
2802        return ret;
2803}
2804
2805static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2806                                        int is_selfpowered)
2807{
2808        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2809        unsigned long flags;
2810
2811        spin_lock_irqsave(&priv_dev->lock, flags);
2812        priv_dev->is_selfpowered = !!is_selfpowered;
2813        spin_unlock_irqrestore(&priv_dev->lock, flags);
2814        return 0;
2815}
2816
2817static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2818{
2819        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2820
2821        if (is_on) {
2822                writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2823        } else {
2824                writel(~0, &priv_dev->regs->ep_ists);
2825                writel(~0, &priv_dev->regs->usb_ists);
2826                writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2827        }
2828
2829        return 0;
2830}
2831
2832static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2833{
2834        struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2835        u32 reg;
2836
2837        cdns3_ep0_config(priv_dev);
2838
2839        /* enable interrupts for endpoint 0 (in and out) */
2840        writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2841
2842        /*
2843         * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2844         * revision of controller.
2845         */
2846        if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2847                reg = readl(&regs->dbg_link1);
2848
2849                reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2850                reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2851                       DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2852                writel(reg, &regs->dbg_link1);
2853        }
2854
2855        /*
2856         * By default some platforms has set protected access to memory.
2857         * This cause problem with cache, so driver restore non-secure
2858         * access to memory.
2859         */
2860        reg = readl(&regs->dma_axi_ctrl);
2861        reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2862               DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2863        writel(reg, &regs->dma_axi_ctrl);
2864
2865        /* enable generic interrupt*/
2866        writel(USB_IEN_INIT, &regs->usb_ien);
2867        writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2868        /*  keep Fast Access bit */
2869        writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2870
2871        cdns3_configure_dmult(priv_dev, NULL);
2872}
2873
2874/**
2875 * cdns3_gadget_udc_start - Gadget start
2876 * @gadget: gadget object
2877 * @driver: driver which operates on this gadget
2878 *
2879 * Returns 0 on success, error code elsewhere
2880 */
2881static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2882                                  struct usb_gadget_driver *driver)
2883{
2884        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2885        unsigned long flags;
2886        enum usb_device_speed max_speed = driver->max_speed;
2887
2888        spin_lock_irqsave(&priv_dev->lock, flags);
2889        priv_dev->gadget_driver = driver;
2890
2891        /* limit speed if necessary */
2892        max_speed = min(driver->max_speed, gadget->max_speed);
2893
2894        switch (max_speed) {
2895        case USB_SPEED_FULL:
2896                writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2897                writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2898                break;
2899        case USB_SPEED_HIGH:
2900                writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2901                break;
2902        case USB_SPEED_SUPER:
2903                break;
2904        default:
2905                dev_err(priv_dev->dev,
2906                        "invalid maximum_speed parameter %d\n",
2907                        max_speed);
2908                fallthrough;
2909        case USB_SPEED_UNKNOWN:
2910                /* default to superspeed */
2911                max_speed = USB_SPEED_SUPER;
2912                break;
2913        }
2914
2915        cdns3_gadget_config(priv_dev);
2916        spin_unlock_irqrestore(&priv_dev->lock, flags);
2917        return 0;
2918}
2919
2920/**
2921 * cdns3_gadget_udc_stop - Stops gadget
2922 * @gadget: gadget object
2923 *
2924 * Returns 0
2925 */
2926static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2927{
2928        struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2929        struct cdns3_endpoint *priv_ep;
2930        u32 bEndpointAddress;
2931        struct usb_ep *ep;
2932        int val;
2933
2934        priv_dev->gadget_driver = NULL;
2935
2936        priv_dev->onchip_used_size = 0;
2937        priv_dev->out_mem_is_allocated = 0;
2938        priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2939
2940        list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2941                priv_ep = ep_to_cdns3_ep(ep);
2942                bEndpointAddress = priv_ep->num | priv_ep->dir;
2943                cdns3_select_ep(priv_dev, bEndpointAddress);
2944                writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2945                readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2946                                          !(val & EP_CMD_EPRST), 1, 100);
2947
2948                priv_ep->flags &= ~EP_CLAIMED;
2949        }
2950
2951        /* disable interrupt for device */
2952        writel(0, &priv_dev->regs->usb_ien);
2953        writel(0, &priv_dev->regs->usb_pwr);
2954        writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2955
2956        return 0;
2957}
2958
2959static const struct usb_gadget_ops cdns3_gadget_ops = {
2960        .get_frame = cdns3_gadget_get_frame,
2961        .wakeup = cdns3_gadget_wakeup,
2962        .set_selfpowered = cdns3_gadget_set_selfpowered,
2963        .pullup = cdns3_gadget_pullup,
2964        .udc_start = cdns3_gadget_udc_start,
2965        .udc_stop = cdns3_gadget_udc_stop,
2966        .match_ep = cdns3_gadget_match_ep,
2967};
2968
2969static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2970{
2971        int i;
2972
2973        /* ep0 OUT point to ep0 IN. */
2974        priv_dev->eps[16] = NULL;
2975
2976        for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2977                if (priv_dev->eps[i]) {
2978                        cdns3_free_trb_pool(priv_dev->eps[i]);
2979                        devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2980                }
2981}
2982
2983/**
2984 * cdns3_init_eps - Initializes software endpoints of gadget
2985 * @priv_dev: extended gadget object
2986 *
2987 * Returns 0 on success, error code elsewhere
2988 */
2989static int cdns3_init_eps(struct cdns3_device *priv_dev)
2990{
2991        u32 ep_enabled_reg, iso_ep_reg;
2992        struct cdns3_endpoint *priv_ep;
2993        int ep_dir, ep_number;
2994        u32 ep_mask;
2995        int ret = 0;
2996        int i;
2997
2998        /* Read it from USB_CAP3 to USB_CAP5 */
2999        ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3000        iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3001
3002        dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3003
3004        for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3005                ep_dir = i >> 4;        /* i div 16 */
3006                ep_number = i & 0xF;    /* i % 16 */
3007                ep_mask = BIT(i);
3008
3009                if (!(ep_enabled_reg & ep_mask))
3010                        continue;
3011
3012                if (ep_dir && !ep_number) {
3013                        priv_dev->eps[i] = priv_dev->eps[0];
3014                        continue;
3015                }
3016
3017                priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3018                                       GFP_KERNEL);
3019                if (!priv_ep)
3020                        goto err;
3021
3022                /* set parent of endpoint object */
3023                priv_ep->cdns3_dev = priv_dev;
3024                priv_dev->eps[i] = priv_ep;
3025                priv_ep->num = ep_number;
3026                priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3027
3028                if (!ep_number) {
3029                        ret = cdns3_init_ep0(priv_dev, priv_ep);
3030                        if (ret) {
3031                                dev_err(priv_dev->dev, "Failed to init ep0\n");
3032                                goto err;
3033                        }
3034                } else {
3035                        snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3036                                 ep_number, !!ep_dir ? "in" : "out");
3037                        priv_ep->endpoint.name = priv_ep->name;
3038
3039                        usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3040                                                   CDNS3_EP_MAX_PACKET_LIMIT);
3041                        priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3042                        priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3043                        if (ep_dir)
3044                                priv_ep->endpoint.caps.dir_in = 1;
3045                        else
3046                                priv_ep->endpoint.caps.dir_out = 1;
3047
3048                        if (iso_ep_reg & ep_mask)
3049                                priv_ep->endpoint.caps.type_iso = 1;
3050
3051                        priv_ep->endpoint.caps.type_bulk = 1;
3052                        priv_ep->endpoint.caps.type_int = 1;
3053
3054                        list_add_tail(&priv_ep->endpoint.ep_list,
3055                                      &priv_dev->gadget.ep_list);
3056                }
3057
3058                priv_ep->flags = 0;
3059
3060                dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",
3061                         priv_ep->name,
3062                         priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3063                         priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3064
3065                INIT_LIST_HEAD(&priv_ep->pending_req_list);
3066                INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3067                INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3068        }
3069
3070        return 0;
3071err:
3072        cdns3_free_all_eps(priv_dev);
3073        return -ENOMEM;
3074}
3075
3076static void cdns3_gadget_release(struct device *dev)
3077{
3078        struct cdns3_device *priv_dev = container_of(dev,
3079                        struct cdns3_device, gadget.dev);
3080
3081        kfree(priv_dev);
3082}
3083
3084static void cdns3_gadget_exit(struct cdns *cdns)
3085{
3086        struct cdns3_device *priv_dev;
3087
3088        priv_dev = cdns->gadget_dev;
3089
3090
3091        pm_runtime_mark_last_busy(cdns->dev);
3092        pm_runtime_put_autosuspend(cdns->dev);
3093
3094        usb_del_gadget(&priv_dev->gadget);
3095        devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3096
3097        cdns3_free_all_eps(priv_dev);
3098
3099        while (!list_empty(&priv_dev->aligned_buf_list)) {
3100                struct cdns3_aligned_buf *buf;
3101
3102                buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3103                dma_free_noncoherent(priv_dev->sysdev, buf->size,
3104                                  buf->buf,
3105                                  buf->dma,
3106                                  buf->dir);
3107
3108                list_del(&buf->list);
3109                kfree(buf);
3110        }
3111
3112        dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3113                          priv_dev->setup_dma);
3114        dma_pool_destroy(priv_dev->eps_dma_pool);
3115
3116        kfree(priv_dev->zlp_buf);
3117        usb_put_gadget(&priv_dev->gadget);
3118        cdns->gadget_dev = NULL;
3119        cdns_drd_gadget_off(cdns);
3120}
3121
3122static int cdns3_gadget_start(struct cdns *cdns)
3123{
3124        struct cdns3_device *priv_dev;
3125        u32 max_speed;
3126        int ret;
3127
3128        priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3129        if (!priv_dev)
3130                return -ENOMEM;
3131
3132        usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3133                        cdns3_gadget_release);
3134        cdns->gadget_dev = priv_dev;
3135        priv_dev->sysdev = cdns->dev;
3136        priv_dev->dev = cdns->dev;
3137        priv_dev->regs = cdns->dev_regs;
3138
3139        device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3140                                 &priv_dev->onchip_buffers);
3141
3142        if (priv_dev->onchip_buffers <=  0) {
3143                u32 reg = readl(&priv_dev->regs->usb_cap2);
3144
3145                priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3146        }
3147
3148        if (!priv_dev->onchip_buffers)
3149                priv_dev->onchip_buffers = 256;
3150
3151        max_speed = usb_get_maximum_speed(cdns->dev);
3152
3153        /* Check the maximum_speed parameter */
3154        switch (max_speed) {
3155        case USB_SPEED_FULL:
3156        case USB_SPEED_HIGH:
3157        case USB_SPEED_SUPER:
3158                break;
3159        default:
3160                dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3161                        max_speed);
3162                fallthrough;
3163        case USB_SPEED_UNKNOWN:
3164                /* default to superspeed */
3165                max_speed = USB_SPEED_SUPER;
3166                break;
3167        }
3168
3169        /* fill gadget fields */
3170        priv_dev->gadget.max_speed = max_speed;
3171        priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3172        priv_dev->gadget.ops = &cdns3_gadget_ops;
3173        priv_dev->gadget.name = "usb-ss-gadget";
3174        priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3175        priv_dev->gadget.irq = cdns->dev_irq;
3176
3177        spin_lock_init(&priv_dev->lock);
3178        INIT_WORK(&priv_dev->pending_status_wq,
3179                  cdns3_pending_setup_status_handler);
3180
3181        INIT_WORK(&priv_dev->aligned_buf_wq,
3182                  cdns3_free_aligned_request_buf);
3183
3184        /* initialize endpoint container */
3185        INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3186        INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3187        priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",
3188                                                 priv_dev->sysdev,
3189                                                 TRB_RING_SIZE, 8, 0);
3190        if (!priv_dev->eps_dma_pool) {
3191                dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");
3192                ret = -ENOMEM;
3193                goto err1;
3194        }
3195
3196        ret = cdns3_init_eps(priv_dev);
3197        if (ret) {
3198                dev_err(priv_dev->dev, "Failed to create endpoints\n");
3199                goto err1;
3200        }
3201
3202        /* allocate memory for setup packet buffer */
3203        priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3204                                                 &priv_dev->setup_dma, GFP_DMA);
3205        if (!priv_dev->setup_buf) {
3206                ret = -ENOMEM;
3207                goto err2;
3208        }
3209
3210        priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3211
3212        dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3213                readl(&priv_dev->regs->usb_cap6));
3214        dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3215                readl(&priv_dev->regs->usb_cap1));
3216        dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3217                readl(&priv_dev->regs->usb_cap2));
3218
3219        priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3220        if (priv_dev->dev_ver >= DEV_VER_V2)
3221                priv_dev->gadget.sg_supported = 1;
3222
3223        priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3224        if (!priv_dev->zlp_buf) {
3225                ret = -ENOMEM;
3226                goto err3;
3227        }
3228
3229        /* add USB gadget device */
3230        ret = usb_add_gadget(&priv_dev->gadget);
3231        if (ret < 0) {
3232                dev_err(priv_dev->dev, "Failed to add gadget\n");
3233                goto err4;
3234        }
3235
3236        return 0;
3237err4:
3238        kfree(priv_dev->zlp_buf);
3239err3:
3240        dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3241                          priv_dev->setup_dma);
3242err2:
3243        cdns3_free_all_eps(priv_dev);
3244err1:
3245        dma_pool_destroy(priv_dev->eps_dma_pool);
3246
3247        usb_put_gadget(&priv_dev->gadget);
3248        cdns->gadget_dev = NULL;
3249        return ret;
3250}
3251
3252static int __cdns3_gadget_init(struct cdns *cdns)
3253{
3254        int ret = 0;
3255
3256        /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3257        ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3258        if (ret) {
3259                dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3260                return ret;
3261        }
3262
3263        cdns_drd_gadget_on(cdns);
3264        pm_runtime_get_sync(cdns->dev);
3265
3266        ret = cdns3_gadget_start(cdns);
3267        if (ret) {
3268                pm_runtime_put_sync(cdns->dev);
3269                return ret;
3270        }
3271
3272        /*
3273         * Because interrupt line can be shared with other components in
3274         * driver it can't use IRQF_ONESHOT flag here.
3275         */
3276        ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3277                                        cdns3_device_irq_handler,
3278                                        cdns3_device_thread_irq_handler,
3279                                        IRQF_SHARED, dev_name(cdns->dev),
3280                                        cdns->gadget_dev);
3281
3282        if (ret)
3283                goto err0;
3284
3285        return 0;
3286err0:
3287        cdns3_gadget_exit(cdns);
3288        return ret;
3289}
3290
3291static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup)
3292__must_hold(&cdns->lock)
3293{
3294        struct cdns3_device *priv_dev = cdns->gadget_dev;
3295
3296        spin_unlock(&cdns->lock);
3297        cdns3_disconnect_gadget(priv_dev);
3298        spin_lock(&cdns->lock);
3299
3300        priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3301        usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3302        cdns3_hw_reset_eps_config(priv_dev);
3303
3304        /* disable interrupt for device */
3305        writel(0, &priv_dev->regs->usb_ien);
3306
3307        return 0;
3308}
3309
3310static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated)
3311{
3312        struct cdns3_device *priv_dev = cdns->gadget_dev;
3313
3314        if (!priv_dev->gadget_driver)
3315                return 0;
3316
3317        cdns3_gadget_config(priv_dev);
3318        if (hibernated)
3319                writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
3320
3321        return 0;
3322}
3323
3324/**
3325 * cdns3_gadget_init - initialize device structure
3326 *
3327 * @cdns: cdns instance
3328 *
3329 * This function initializes the gadget.
3330 */
3331int cdns3_gadget_init(struct cdns *cdns)
3332{
3333        struct cdns_role_driver *rdrv;
3334
3335        rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3336        if (!rdrv)
3337                return -ENOMEM;
3338
3339        rdrv->start     = __cdns3_gadget_init;
3340        rdrv->stop      = cdns3_gadget_exit;
3341        rdrv->suspend   = cdns3_gadget_suspend;
3342        rdrv->resume    = cdns3_gadget_resume;
3343        rdrv->state     = CDNS_ROLE_STATE_INACTIVE;
3344        rdrv->name      = "gadget";
3345        cdns->roles[USB_ROLE_DEVICE] = rdrv;
3346
3347        return 0;
3348}
3349