linux/drivers/usb/cdns3/cdnsp-gadget.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence CDNSP DRD Driver.
   4 *
   5 * Copyright (C) 2020 Cadence.
   6 *
   7 * Author: Pawel Laszczak <pawell@cadence.com>
   8 *
   9 */
  10
  11#include <linux/moduleparam.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/module.h>
  14#include <linux/iopoll.h>
  15#include <linux/delay.h>
  16#include <linux/log2.h>
  17#include <linux/slab.h>
  18#include <linux/pci.h>
  19#include <linux/irq.h>
  20#include <linux/dmi.h>
  21
  22#include "core.h"
  23#include "gadget-export.h"
  24#include "drd.h"
  25#include "cdnsp-gadget.h"
  26#include "cdnsp-trace.h"
  27
  28unsigned int cdnsp_port_speed(unsigned int port_status)
  29{
  30        /*Detect gadget speed based on PORTSC register*/
  31        if (DEV_SUPERSPEEDPLUS(port_status))
  32                return USB_SPEED_SUPER_PLUS;
  33        else if (DEV_SUPERSPEED(port_status))
  34                return USB_SPEED_SUPER;
  35        else if (DEV_HIGHSPEED(port_status))
  36                return USB_SPEED_HIGH;
  37        else if (DEV_FULLSPEED(port_status))
  38                return USB_SPEED_FULL;
  39
  40        /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
  41        return USB_SPEED_UNKNOWN;
  42}
  43
  44/*
  45 * Given a port state, this function returns a value that would result in the
  46 * port being in the same state, if the value was written to the port status
  47 * control register.
  48 * Save Read Only (RO) bits and save read/write bits where
  49 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
  50 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
  51 */
  52u32 cdnsp_port_state_to_neutral(u32 state)
  53{
  54        /* Save read-only status and port state. */
  55        return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
  56}
  57
  58/**
  59 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
  60 *                           with capability ID id.
  61 * @base: PCI MMIO registers base address.
  62 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
  63 *         beginning of list)
  64 * @id: Extended capability ID to search for.
  65 *
  66 * Returns the offset of the next matching extended capability structure.
  67 * Some capabilities can occur several times,
  68 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
  69 */
  70int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
  71{
  72        u32 offset = start;
  73        u32 next;
  74        u32 val;
  75
  76        if (!start || start == HCC_PARAMS_OFFSET) {
  77                val = readl(base + HCC_PARAMS_OFFSET);
  78                if (val == ~0)
  79                        return 0;
  80
  81                offset = HCC_EXT_CAPS(val) << 2;
  82                if (!offset)
  83                        return 0;
  84        };
  85
  86        do {
  87                val = readl(base + offset);
  88                if (val == ~0)
  89                        return 0;
  90
  91                if (EXT_CAPS_ID(val) == id && offset != start)
  92                        return offset;
  93
  94                next = EXT_CAPS_NEXT(val);
  95                offset += next << 2;
  96        } while (next);
  97
  98        return 0;
  99}
 100
 101void cdnsp_set_link_state(struct cdnsp_device *pdev,
 102                          __le32 __iomem *port_regs,
 103                          u32 link_state)
 104{
 105        int port_num = 0xFF;
 106        u32 temp;
 107
 108        temp = readl(port_regs);
 109        temp = cdnsp_port_state_to_neutral(temp);
 110        temp |= PORT_WKCONN_E | PORT_WKDISC_E;
 111        writel(temp, port_regs);
 112
 113        temp &= ~PORT_PLS_MASK;
 114        temp |= PORT_LINK_STROBE | link_state;
 115
 116        if (pdev->active_port)
 117                port_num = pdev->active_port->port_num;
 118
 119        trace_cdnsp_handle_port_status(port_num, readl(port_regs));
 120        writel(temp, port_regs);
 121        trace_cdnsp_link_state_changed(port_num, readl(port_regs));
 122}
 123
 124static void cdnsp_disable_port(struct cdnsp_device *pdev,
 125                               __le32 __iomem *port_regs)
 126{
 127        u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
 128
 129        writel(temp | PORT_PED, port_regs);
 130}
 131
 132static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
 133                                        __le32 __iomem *port_regs)
 134{
 135        u32 portsc = readl(port_regs);
 136
 137        writel(cdnsp_port_state_to_neutral(portsc) |
 138               (portsc & PORT_CHANGE_BITS), port_regs);
 139}
 140
 141static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 142{
 143        __le32 __iomem *reg;
 144        void __iomem *base;
 145        u32 offset = 0;
 146
 147        base = &pdev->cap_regs->hc_capbase;
 148        offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 149        reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 150
 151        bit = readl(reg) | bit;
 152        writel(bit, reg);
 153}
 154
 155static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 156{
 157        __le32 __iomem *reg;
 158        void __iomem *base;
 159        u32 offset = 0;
 160
 161        base = &pdev->cap_regs->hc_capbase;
 162        offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 163        reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 164
 165        bit = readl(reg) & ~bit;
 166        writel(bit, reg);
 167}
 168
 169/*
 170 * Disable interrupts and begin the controller halting process.
 171 */
 172static void cdnsp_quiesce(struct cdnsp_device *pdev)
 173{
 174        u32 halted;
 175        u32 mask;
 176        u32 cmd;
 177
 178        mask = ~(u32)(CDNSP_IRQS);
 179
 180        halted = readl(&pdev->op_regs->status) & STS_HALT;
 181        if (!halted)
 182                mask &= ~(CMD_R_S | CMD_DEVEN);
 183
 184        cmd = readl(&pdev->op_regs->command);
 185        cmd &= mask;
 186        writel(cmd, &pdev->op_regs->command);
 187}
 188
 189/*
 190 * Force controller into halt state.
 191 *
 192 * Disable any IRQs and clear the run/stop bit.
 193 * Controller will complete any current and actively pipelined transactions, and
 194 * should halt within 16 ms of the run/stop bit being cleared.
 195 * Read controller Halted bit in the status register to see when the
 196 * controller is finished.
 197 */
 198int cdnsp_halt(struct cdnsp_device *pdev)
 199{
 200        int ret;
 201        u32 val;
 202
 203        cdnsp_quiesce(pdev);
 204
 205        ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
 206                                        val & STS_HALT, 1,
 207                                        CDNSP_MAX_HALT_USEC);
 208        if (ret) {
 209                dev_err(pdev->dev, "ERROR: Device halt failed\n");
 210                return ret;
 211        }
 212
 213        pdev->cdnsp_state |= CDNSP_STATE_HALTED;
 214
 215        return 0;
 216}
 217
 218/*
 219 * device controller died, register read returns 0xffffffff, or command never
 220 * ends.
 221 */
 222void cdnsp_died(struct cdnsp_device *pdev)
 223{
 224        dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
 225        pdev->cdnsp_state |= CDNSP_STATE_DYING;
 226        cdnsp_halt(pdev);
 227}
 228
 229/*
 230 * Set the run bit and wait for the device to be running.
 231 */
 232static int cdnsp_start(struct cdnsp_device *pdev)
 233{
 234        u32 temp;
 235        int ret;
 236
 237        temp = readl(&pdev->op_regs->command);
 238        temp |= (CMD_R_S | CMD_DEVEN);
 239        writel(temp, &pdev->op_regs->command);
 240
 241        pdev->cdnsp_state = 0;
 242
 243        /*
 244         * Wait for the STS_HALT Status bit to be 0 to indicate the device is
 245         * running.
 246         */
 247        ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 248                                        !(temp & STS_HALT), 1,
 249                                        CDNSP_MAX_HALT_USEC);
 250        if (ret) {
 251                pdev->cdnsp_state = CDNSP_STATE_DYING;
 252                dev_err(pdev->dev, "ERROR: Controller run failed\n");
 253        }
 254
 255        return ret;
 256}
 257
 258/*
 259 * Reset a halted controller.
 260 *
 261 * This resets pipelines, timers, counters, state machines, etc.
 262 * Transactions will be terminated immediately, and operational registers
 263 * will be set to their defaults.
 264 */
 265int cdnsp_reset(struct cdnsp_device *pdev)
 266{
 267        u32 command;
 268        u32 temp;
 269        int ret;
 270
 271        temp = readl(&pdev->op_regs->status);
 272
 273        if (temp == ~(u32)0) {
 274                dev_err(pdev->dev, "Device not accessible, reset failed.\n");
 275                return -ENODEV;
 276        }
 277
 278        if ((temp & STS_HALT) == 0) {
 279                dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
 280                return -EINVAL;
 281        }
 282
 283        command = readl(&pdev->op_regs->command);
 284        command |= CMD_RESET;
 285        writel(command, &pdev->op_regs->command);
 286
 287        ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
 288                                        !(temp & CMD_RESET), 1,
 289                                        10 * 1000);
 290        if (ret) {
 291                dev_err(pdev->dev, "ERROR: Controller reset failed\n");
 292                return ret;
 293        }
 294
 295        /*
 296         * CDNSP cannot write any doorbells or operational registers other
 297         * than status until the "Controller Not Ready" flag is cleared.
 298         */
 299        ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 300                                        !(temp & STS_CNR), 1,
 301                                        10 * 1000);
 302
 303        if (ret) {
 304                dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
 305                return ret;
 306        }
 307
 308        dev_dbg(pdev->dev, "Controller ready to work");
 309
 310        return ret;
 311}
 312
 313/*
 314 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
 315 * descriptor.Use the return value to right shift 1 for the bitmask.
 316 *
 317 * Index = (epnum * 2) + direction - 1,
 318 * where direction = 0 for OUT, 1 for IN.
 319 * For control endpoints, the IN index is used (OUT index is unused), so
 320 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
 321 */
 322static unsigned int
 323        cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
 324{
 325        unsigned int index = (unsigned int)usb_endpoint_num(desc);
 326
 327        if (usb_endpoint_xfer_control(desc))
 328                return index * 2;
 329
 330        return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
 331}
 332
 333/*
 334 * Find the flag for this endpoint (for use in the control context). Use the
 335 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
 336 * bit 1, etc.
 337 */
 338static unsigned int
 339        cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
 340{
 341        return 1 << (cdnsp_get_endpoint_index(desc) + 1);
 342}
 343
 344int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 345{
 346        struct cdnsp_device *pdev = pep->pdev;
 347        struct usb_request *request;
 348        int ret;
 349
 350        if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
 351                trace_cdnsp_request_enqueue_busy(preq);
 352                return -EBUSY;
 353        }
 354
 355        request = &preq->request;
 356        request->actual = 0;
 357        request->status = -EINPROGRESS;
 358        preq->direction = pep->direction;
 359        preq->epnum = pep->number;
 360        preq->td.drbl = 0;
 361
 362        ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
 363        if (ret) {
 364                trace_cdnsp_request_enqueue_error(preq);
 365                return ret;
 366        }
 367
 368        list_add_tail(&preq->list, &pep->pending_list);
 369
 370        trace_cdnsp_request_enqueue(preq);
 371
 372        switch (usb_endpoint_type(pep->endpoint.desc)) {
 373        case USB_ENDPOINT_XFER_CONTROL:
 374                ret = cdnsp_queue_ctrl_tx(pdev, preq);
 375                break;
 376        case USB_ENDPOINT_XFER_BULK:
 377        case USB_ENDPOINT_XFER_INT:
 378                ret = cdnsp_queue_bulk_tx(pdev, preq);
 379                break;
 380        case USB_ENDPOINT_XFER_ISOC:
 381                ret = cdnsp_queue_isoc_tx_prepare(pdev, preq);
 382        }
 383
 384        if (ret)
 385                goto unmap;
 386
 387        return 0;
 388
 389unmap:
 390        usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
 391                                        pep->direction);
 392        list_del(&preq->list);
 393        trace_cdnsp_request_enqueue_error(preq);
 394
 395        return ret;
 396}
 397
 398/*
 399 * Remove the request's TD from the endpoint ring. This may cause the
 400 * controller to stop USB transfers, potentially stopping in the middle of a
 401 * TRB buffer. The controller should pick up where it left off in the TD,
 402 * unless a Set Transfer Ring Dequeue Pointer is issued.
 403 *
 404 * The TRBs that make up the buffers for the canceled request will be "removed"
 405 * from the ring. Since the ring is a contiguous structure, they can't be
 406 * physically removed. Instead, there are two options:
 407 *
 408 *  1) If the controller is in the middle of processing the request to be
 409 *     canceled, we simply move the ring's dequeue pointer past those TRBs
 410 *     using the Set Transfer Ring Dequeue Pointer command. This will be
 411 *     the common case, when drivers timeout on the last submitted request
 412 *     and attempt to cancel.
 413 *
 414 *  2) If the controller is in the middle of a different TD, we turn the TRBs
 415 *     into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
 416 *     The controller will need to invalidate the any TRBs it has cached after
 417 *     the stop endpoint command.
 418 *
 419 *  3) The TD may have completed by the time the Stop Endpoint Command
 420 *     completes, so software needs to handle that case too.
 421 *
 422 */
 423int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 424{
 425        struct cdnsp_device *pdev = pep->pdev;
 426        int ret_stop = 0;
 427        int ret_rem;
 428
 429        trace_cdnsp_request_dequeue(preq);
 430
 431        if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
 432                ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
 433
 434        ret_rem = cdnsp_remove_request(pdev, preq, pep);
 435
 436        return ret_rem ? ret_rem : ret_stop;
 437}
 438
 439static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
 440{
 441        struct cdnsp_input_control_ctx *ctrl_ctx;
 442        struct cdnsp_slot_ctx *slot_ctx;
 443        struct cdnsp_ep_ctx *ep_ctx;
 444        int i;
 445
 446        ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 447
 448        /*
 449         * When a device's add flag and drop flag are zero, any subsequent
 450         * configure endpoint command will leave that endpoint's state
 451         * untouched. Make sure we don't leave any old state in the input
 452         * endpoint contexts.
 453         */
 454        ctrl_ctx->drop_flags = 0;
 455        ctrl_ctx->add_flags = 0;
 456        slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 457        slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 458
 459        /* Endpoint 0 is always valid */
 460        slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
 461        for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
 462                ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
 463                ep_ctx->ep_info = 0;
 464                ep_ctx->ep_info2 = 0;
 465                ep_ctx->deq = 0;
 466                ep_ctx->tx_info = 0;
 467        }
 468}
 469
 470/* Issue a configure endpoint command and wait for it to finish. */
 471static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
 472{
 473        int ret;
 474
 475        cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
 476        cdnsp_ring_cmd_db(pdev);
 477        ret = cdnsp_wait_for_cmd_compl(pdev);
 478        if (ret) {
 479                dev_err(pdev->dev,
 480                        "ERR: unexpected command completion code 0x%x.\n", ret);
 481                return -EINVAL;
 482        }
 483
 484        return ret;
 485}
 486
 487static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
 488                                       struct cdnsp_ep *pep)
 489{
 490        struct cdnsp_segment *segment;
 491        union cdnsp_trb *event;
 492        u32 cycle_state;
 493        u32  data;
 494
 495        event = pdev->event_ring->dequeue;
 496        segment = pdev->event_ring->deq_seg;
 497        cycle_state = pdev->event_ring->cycle_state;
 498
 499        while (1) {
 500                data = le32_to_cpu(event->trans_event.flags);
 501
 502                /* Check the owner of the TRB. */
 503                if ((data & TRB_CYCLE) != cycle_state)
 504                        break;
 505
 506                if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
 507                    TRB_TO_EP_ID(data) == (pep->idx + 1)) {
 508                        data |= TRB_EVENT_INVALIDATE;
 509                        event->trans_event.flags = cpu_to_le32(data);
 510                }
 511
 512                if (cdnsp_last_trb_on_seg(segment, event)) {
 513                        cycle_state ^= 1;
 514                        segment = pdev->event_ring->deq_seg->next;
 515                        event = segment->trbs;
 516                } else {
 517                        event++;
 518                }
 519        }
 520}
 521
 522int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
 523{
 524        struct cdnsp_segment *event_deq_seg;
 525        union cdnsp_trb *cmd_trb;
 526        dma_addr_t cmd_deq_dma;
 527        union cdnsp_trb *event;
 528        u32 cycle_state;
 529        int ret, val;
 530        u64 cmd_dma;
 531        u32  flags;
 532
 533        cmd_trb = pdev->cmd.command_trb;
 534        pdev->cmd.status = 0;
 535
 536        trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
 537
 538        ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
 539                                        !CMD_RING_BUSY(val), 1,
 540                                        CDNSP_CMD_TIMEOUT);
 541        if (ret) {
 542                dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
 543                trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
 544                pdev->cdnsp_state = CDNSP_STATE_DYING;
 545                return -ETIMEDOUT;
 546        }
 547
 548        event = pdev->event_ring->dequeue;
 549        event_deq_seg = pdev->event_ring->deq_seg;
 550        cycle_state = pdev->event_ring->cycle_state;
 551
 552        cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
 553        if (!cmd_deq_dma)
 554                return -EINVAL;
 555
 556        while (1) {
 557                flags = le32_to_cpu(event->event_cmd.flags);
 558
 559                /* Check the owner of the TRB. */
 560                if ((flags & TRB_CYCLE) != cycle_state)
 561                        return -EINVAL;
 562
 563                cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
 564
 565                /*
 566                 * Check whether the completion event is for last queued
 567                 * command.
 568                 */
 569                if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
 570                    cmd_dma != (u64)cmd_deq_dma) {
 571                        if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
 572                                event++;
 573                                continue;
 574                        }
 575
 576                        if (cdnsp_last_trb_on_ring(pdev->event_ring,
 577                                                   event_deq_seg, event))
 578                                cycle_state ^= 1;
 579
 580                        event_deq_seg = event_deq_seg->next;
 581                        event = event_deq_seg->trbs;
 582                        continue;
 583                }
 584
 585                trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
 586
 587                pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
 588                if (pdev->cmd.status == COMP_SUCCESS)
 589                        return 0;
 590
 591                return -pdev->cmd.status;
 592        }
 593}
 594
 595int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
 596                        struct cdnsp_ep *pep,
 597                        int value)
 598{
 599        int ret;
 600
 601        trace_cdnsp_ep_halt(value ? "Set" : "Clear");
 602
 603        if (value) {
 604                ret = cdnsp_cmd_stop_ep(pdev, pep);
 605                if (ret)
 606                        return ret;
 607
 608                if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
 609                        cdnsp_queue_halt_endpoint(pdev, pep->idx);
 610                        cdnsp_ring_cmd_db(pdev);
 611                        ret = cdnsp_wait_for_cmd_compl(pdev);
 612                }
 613
 614                pep->ep_state |= EP_HALTED;
 615        } else {
 616                /*
 617                 * In device mode driver can call reset endpoint command
 618                 * from any endpoint state.
 619                 */
 620                cdnsp_queue_reset_ep(pdev, pep->idx);
 621                cdnsp_ring_cmd_db(pdev);
 622                ret = cdnsp_wait_for_cmd_compl(pdev);
 623                trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
 624
 625                if (ret)
 626                        return ret;
 627
 628                pep->ep_state &= ~EP_HALTED;
 629
 630                if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
 631                        cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 632
 633                pep->ep_state &= ~EP_WEDGE;
 634        }
 635
 636        return 0;
 637}
 638
 639static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
 640                                          struct cdnsp_ep *pep)
 641{
 642        struct cdnsp_input_control_ctx *ctrl_ctx;
 643        struct cdnsp_slot_ctx *slot_ctx;
 644        int ret = 0;
 645        u32 ep_sts;
 646        int i;
 647
 648        ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 649
 650        /* Don't issue the command if there's no endpoints to update. */
 651        if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
 652                return 0;
 653
 654        ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
 655        ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
 656        ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
 657
 658        /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
 659        slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 660        for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
 661                __le32 le32 = cpu_to_le32(BIT(i));
 662
 663                if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
 664                    (ctrl_ctx->add_flags & le32) || i == 1) {
 665                        slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 666                        slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
 667                        break;
 668                }
 669        }
 670
 671        ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
 672
 673        if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
 674             ep_sts == EP_STATE_DISABLED) ||
 675            (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
 676                ret = cdnsp_configure_endpoint(pdev);
 677
 678        trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
 679        trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
 680
 681        cdnsp_zero_in_ctx(pdev);
 682
 683        return ret;
 684}
 685
 686/*
 687 * This submits a Reset Device Command, which will set the device state to 0,
 688 * set the device address to 0, and disable all the endpoints except the default
 689 * control endpoint. The USB core should come back and call
 690 * cdnsp_setup_device(), and then re-set up the configuration.
 691 */
 692int cdnsp_reset_device(struct cdnsp_device *pdev)
 693{
 694        struct cdnsp_slot_ctx *slot_ctx;
 695        int slot_state;
 696        int ret, i;
 697
 698        slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 699        slot_ctx->dev_info = 0;
 700        pdev->device_address = 0;
 701
 702        /* If device is not setup, there is no point in resetting it. */
 703        slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 704        slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 705        trace_cdnsp_reset_device(slot_ctx);
 706
 707        if (slot_state <= SLOT_STATE_DEFAULT &&
 708            pdev->eps[0].ep_state & EP_HALTED) {
 709                cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
 710        }
 711
 712        /*
 713         * During Reset Device command controller shall transition the
 714         * endpoint ep0 to the Running State.
 715         */
 716        pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
 717        pdev->eps[0].ep_state |= EP_ENABLED;
 718
 719        if (slot_state <= SLOT_STATE_DEFAULT)
 720                return 0;
 721
 722        cdnsp_queue_reset_device(pdev);
 723        cdnsp_ring_cmd_db(pdev);
 724        ret = cdnsp_wait_for_cmd_compl(pdev);
 725
 726        /*
 727         * After Reset Device command all not default endpoints
 728         * are in Disabled state.
 729         */
 730        for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
 731                pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
 732
 733        trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
 734
 735        if (ret)
 736                dev_err(pdev->dev, "Reset device failed with error code %d",
 737                        ret);
 738
 739        return ret;
 740}
 741
 742/*
 743 * Sets the MaxPStreams field and the Linear Stream Array field.
 744 * Sets the dequeue pointer to the stream context array.
 745 */
 746static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
 747                                             struct cdnsp_ep_ctx *ep_ctx,
 748                                             struct cdnsp_stream_info *stream_info)
 749{
 750        u32 max_primary_streams;
 751
 752        /* MaxPStreams is the number of stream context array entries, not the
 753         * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
 754         * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
 755         */
 756        max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
 757        ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
 758        ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
 759                                       | EP_HAS_LSA);
 760        ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
 761}
 762
 763/*
 764 * The drivers use this function to prepare a bulk endpoints to use streams.
 765 *
 766 * Don't allow the call to succeed if endpoint only supports one stream
 767 * (which means it doesn't support streams at all).
 768 */
 769int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
 770{
 771        unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
 772        unsigned int num_stream_ctxs;
 773        int ret;
 774
 775        if (num_streams ==  0)
 776                return 0;
 777
 778        if (num_streams > STREAM_NUM_STREAMS)
 779                return -EINVAL;
 780
 781        /*
 782         * Add two to the number of streams requested to account for
 783         * stream 0 that is reserved for controller usage and one additional
 784         * for TASK SET FULL response.
 785         */
 786        num_streams += 2;
 787
 788        /* The stream context array size must be a power of two */
 789        num_stream_ctxs = roundup_pow_of_two(num_streams);
 790
 791        trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
 792
 793        ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
 794        if (ret)
 795                return ret;
 796
 797        cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
 798
 799        pep->ep_state |= EP_HAS_STREAMS;
 800        pep->stream_info.td_count = 0;
 801        pep->stream_info.first_prime_det = 0;
 802
 803        /* Subtract 1 for stream 0, which drivers can't use. */
 804        return num_streams - 1;
 805}
 806
 807int cdnsp_disable_slot(struct cdnsp_device *pdev)
 808{
 809        int ret;
 810
 811        cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
 812        cdnsp_ring_cmd_db(pdev);
 813        ret = cdnsp_wait_for_cmd_compl(pdev);
 814
 815        pdev->slot_id = 0;
 816        pdev->active_port = NULL;
 817
 818        trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 819
 820        memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
 821        memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
 822
 823        return ret;
 824}
 825
 826int cdnsp_enable_slot(struct cdnsp_device *pdev)
 827{
 828        struct cdnsp_slot_ctx *slot_ctx;
 829        int slot_state;
 830        int ret;
 831
 832        /* If device is not setup, there is no point in resetting it */
 833        slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 834        slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 835
 836        if (slot_state != SLOT_STATE_DISABLED)
 837                return 0;
 838
 839        cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
 840        cdnsp_ring_cmd_db(pdev);
 841        ret = cdnsp_wait_for_cmd_compl(pdev);
 842        if (ret)
 843                goto show_trace;
 844
 845        pdev->slot_id = 1;
 846
 847show_trace:
 848        trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 849
 850        return ret;
 851}
 852
 853/*
 854 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
 855 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
 856 */
 857int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
 858{
 859        struct cdnsp_input_control_ctx *ctrl_ctx;
 860        struct cdnsp_slot_ctx *slot_ctx;
 861        int dev_state = 0;
 862        int ret;
 863
 864        if (!pdev->slot_id) {
 865                trace_cdnsp_slot_id("incorrect");
 866                return -EINVAL;
 867        }
 868
 869        if (!pdev->active_port->port_num)
 870                return -EINVAL;
 871
 872        slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 873        dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 874
 875        if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
 876                trace_cdnsp_slot_already_in_default(slot_ctx);
 877                return 0;
 878        }
 879
 880        slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 881        ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 882
 883        if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
 884                ret = cdnsp_setup_addressable_priv_dev(pdev);
 885                if (ret)
 886                        return ret;
 887        }
 888
 889        cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
 890
 891        ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
 892        ctrl_ctx->drop_flags = 0;
 893
 894        trace_cdnsp_setup_device_slot(slot_ctx);
 895
 896        cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
 897        cdnsp_ring_cmd_db(pdev);
 898        ret = cdnsp_wait_for_cmd_compl(pdev);
 899
 900        trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
 901
 902        /* Zero the input context control for later use. */
 903        ctrl_ctx->add_flags = 0;
 904        ctrl_ctx->drop_flags = 0;
 905
 906        return ret;
 907}
 908
 909void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
 910                                 struct usb_request *req,
 911                                 int enable)
 912{
 913        if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
 914                return;
 915
 916        trace_cdnsp_lpm(enable);
 917
 918        if (enable)
 919                writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
 920                       &pdev->active_port->regs->portpmsc);
 921        else
 922                writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
 923}
 924
 925static int cdnsp_get_frame(struct cdnsp_device *pdev)
 926{
 927        return readl(&pdev->run_regs->microframe_index) >> 3;
 928}
 929
 930static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
 931                                  const struct usb_endpoint_descriptor *desc)
 932{
 933        struct cdnsp_input_control_ctx *ctrl_ctx;
 934        struct cdnsp_device *pdev;
 935        struct cdnsp_ep *pep;
 936        unsigned long flags;
 937        u32 added_ctxs;
 938        int ret;
 939
 940        if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
 941            !desc->wMaxPacketSize)
 942                return -EINVAL;
 943
 944        pep = to_cdnsp_ep(ep);
 945        pdev = pep->pdev;
 946        pep->ep_state &= ~EP_UNCONFIGURED;
 947
 948        if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
 949                          "%s is already enabled\n", pep->name))
 950                return 0;
 951
 952        spin_lock_irqsave(&pdev->lock, flags);
 953
 954        added_ctxs = cdnsp_get_endpoint_flag(desc);
 955        if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
 956                dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
 957                ret = -EINVAL;
 958                goto unlock;
 959        }
 960
 961        pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
 962
 963        if (pdev->gadget.speed == USB_SPEED_FULL) {
 964                if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
 965                        pep->interval = desc->bInterval << 3;
 966                if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
 967                        pep->interval = BIT(desc->bInterval - 1) << 3;
 968        }
 969
 970        if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
 971                if (pep->interval > BIT(12)) {
 972                        dev_err(pdev->dev, "bInterval %d not supported\n",
 973                                desc->bInterval);
 974                        ret = -EINVAL;
 975                        goto unlock;
 976                }
 977                cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
 978        }
 979
 980        ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
 981        if (ret)
 982                goto unlock;
 983
 984        ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 985        ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
 986        ctrl_ctx->drop_flags = 0;
 987
 988        ret = cdnsp_update_eps_configuration(pdev, pep);
 989        if (ret) {
 990                cdnsp_free_endpoint_rings(pdev, pep);
 991                goto unlock;
 992        }
 993
 994        pep->ep_state |= EP_ENABLED;
 995        pep->ep_state &= ~EP_STOPPED;
 996
 997unlock:
 998        trace_cdnsp_ep_enable_end(pep, 0);
 999        spin_unlock_irqrestore(&pdev->lock, flags);
1000
1001        return ret;
1002}
1003
1004static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1005{
1006        struct cdnsp_input_control_ctx *ctrl_ctx;
1007        struct cdnsp_request *preq;
1008        struct cdnsp_device *pdev;
1009        struct cdnsp_ep *pep;
1010        unsigned long flags;
1011        u32 drop_flag;
1012        int ret = 0;
1013
1014        if (!ep)
1015                return -EINVAL;
1016
1017        pep = to_cdnsp_ep(ep);
1018        pdev = pep->pdev;
1019
1020        spin_lock_irqsave(&pdev->lock, flags);
1021
1022        if (!(pep->ep_state & EP_ENABLED)) {
1023                dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1024                ret = -EINVAL;
1025                goto finish;
1026        }
1027
1028        pep->ep_state |= EP_DIS_IN_RROGRESS;
1029
1030        /* Endpoint was unconfigured by Reset Device command. */
1031        if (!(pep->ep_state & EP_UNCONFIGURED)) {
1032                cdnsp_cmd_stop_ep(pdev, pep);
1033                cdnsp_cmd_flush_ep(pdev, pep);
1034        }
1035
1036        /* Remove all queued USB requests. */
1037        while (!list_empty(&pep->pending_list)) {
1038                preq = next_request(&pep->pending_list);
1039                cdnsp_ep_dequeue(pep, preq);
1040        }
1041
1042        cdnsp_invalidate_ep_events(pdev, pep);
1043
1044        pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1045        drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1046        ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1047        ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1048        ctrl_ctx->add_flags = 0;
1049
1050        cdnsp_endpoint_zero(pdev, pep);
1051
1052        if (!(pep->ep_state & EP_UNCONFIGURED))
1053                ret = cdnsp_update_eps_configuration(pdev, pep);
1054
1055        cdnsp_free_endpoint_rings(pdev, pep);
1056
1057        pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1058        pep->ep_state |= EP_STOPPED;
1059
1060finish:
1061        trace_cdnsp_ep_disable_end(pep, 0);
1062        spin_unlock_irqrestore(&pdev->lock, flags);
1063
1064        return ret;
1065}
1066
1067static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1068                                                         gfp_t gfp_flags)
1069{
1070        struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1071        struct cdnsp_request *preq;
1072
1073        preq = kzalloc(sizeof(*preq), gfp_flags);
1074        if (!preq)
1075                return NULL;
1076
1077        preq->epnum = pep->number;
1078        preq->pep = pep;
1079
1080        trace_cdnsp_alloc_request(preq);
1081
1082        return &preq->request;
1083}
1084
1085static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1086                                         struct usb_request *request)
1087{
1088        struct cdnsp_request *preq = to_cdnsp_request(request);
1089
1090        trace_cdnsp_free_request(preq);
1091        kfree(preq);
1092}
1093
1094static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1095                                 struct usb_request *request,
1096                                 gfp_t gfp_flags)
1097{
1098        struct cdnsp_request *preq;
1099        struct cdnsp_device *pdev;
1100        struct cdnsp_ep *pep;
1101        unsigned long flags;
1102        int ret;
1103
1104        if (!request || !ep)
1105                return -EINVAL;
1106
1107        pep = to_cdnsp_ep(ep);
1108        pdev = pep->pdev;
1109
1110        if (!(pep->ep_state & EP_ENABLED)) {
1111                dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1112                        pep->name);
1113                return -EINVAL;
1114        }
1115
1116        preq = to_cdnsp_request(request);
1117        spin_lock_irqsave(&pdev->lock, flags);
1118        ret = cdnsp_ep_enqueue(pep, preq);
1119        spin_unlock_irqrestore(&pdev->lock, flags);
1120
1121        return ret;
1122}
1123
1124static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1125                                   struct usb_request *request)
1126{
1127        struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1128        struct cdnsp_device *pdev = pep->pdev;
1129        unsigned long flags;
1130        int ret;
1131
1132        if (!pep->endpoint.desc) {
1133                dev_err(pdev->dev,
1134                        "%s: can't dequeue to disabled endpoint\n",
1135                        pep->name);
1136                return -ESHUTDOWN;
1137        }
1138
1139        /* Requests has been dequeued during disabling endpoint. */
1140        if (!(pep->ep_state & EP_ENABLED))
1141                return 0;
1142
1143        spin_lock_irqsave(&pdev->lock, flags);
1144        ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1145        spin_unlock_irqrestore(&pdev->lock, flags);
1146
1147        return ret;
1148}
1149
1150static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1151{
1152        struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1153        struct cdnsp_device *pdev = pep->pdev;
1154        struct cdnsp_request *preq;
1155        unsigned long flags;
1156        int ret;
1157
1158        spin_lock_irqsave(&pdev->lock, flags);
1159
1160        preq = next_request(&pep->pending_list);
1161        if (value) {
1162                if (preq) {
1163                        trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1164                        ret = -EAGAIN;
1165                        goto done;
1166                }
1167        }
1168
1169        ret = cdnsp_halt_endpoint(pdev, pep, value);
1170
1171done:
1172        spin_unlock_irqrestore(&pdev->lock, flags);
1173        return ret;
1174}
1175
1176static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1177{
1178        struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1179        struct cdnsp_device *pdev = pep->pdev;
1180        unsigned long flags;
1181        int ret;
1182
1183        spin_lock_irqsave(&pdev->lock, flags);
1184        pep->ep_state |= EP_WEDGE;
1185        ret = cdnsp_halt_endpoint(pdev, pep, 1);
1186        spin_unlock_irqrestore(&pdev->lock, flags);
1187
1188        return ret;
1189}
1190
1191static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1192        .enable         = cdnsp_gadget_ep_enable,
1193        .disable        = cdnsp_gadget_ep_disable,
1194        .alloc_request  = cdnsp_gadget_ep_alloc_request,
1195        .free_request   = cdnsp_gadget_ep_free_request,
1196        .queue          = cdnsp_gadget_ep_queue,
1197        .dequeue        = cdnsp_gadget_ep_dequeue,
1198        .set_halt       = cdnsp_gadget_ep_set_halt,
1199        .set_wedge      = cdnsp_gadget_ep_set_wedge,
1200};
1201
1202static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1203        .enable         = cdnsp_gadget_ep_enable,
1204        .disable        = cdnsp_gadget_ep_disable,
1205        .alloc_request  = cdnsp_gadget_ep_alloc_request,
1206        .free_request   = cdnsp_gadget_ep_free_request,
1207        .queue          = cdnsp_gadget_ep_queue,
1208        .dequeue        = cdnsp_gadget_ep_dequeue,
1209        .set_halt       = cdnsp_gadget_ep_set_halt,
1210        .set_wedge      = cdnsp_gadget_ep_set_wedge,
1211};
1212
1213void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1214                           struct cdnsp_request *preq,
1215                           int status)
1216{
1217        struct cdnsp_device *pdev = pep->pdev;
1218
1219        list_del(&preq->list);
1220
1221        if (preq->request.status == -EINPROGRESS)
1222                preq->request.status = status;
1223
1224        usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1225                                        preq->direction);
1226
1227        trace_cdnsp_request_giveback(preq);
1228
1229        if (preq != &pdev->ep0_preq) {
1230                spin_unlock(&pdev->lock);
1231                usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1232                spin_lock(&pdev->lock);
1233        }
1234}
1235
1236static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1237        .bLength =              USB_DT_ENDPOINT_SIZE,
1238        .bDescriptorType =      USB_DT_ENDPOINT,
1239        .bmAttributes =         USB_ENDPOINT_XFER_CONTROL,
1240};
1241
1242static int cdnsp_run(struct cdnsp_device *pdev,
1243                     enum usb_device_speed speed)
1244{
1245        u32 fs_speed = 0;
1246        u64 temp_64;
1247        u32 temp;
1248        int ret;
1249
1250        temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1251        temp_64 &= ~ERST_PTR_MASK;
1252        temp = readl(&pdev->ir_set->irq_control);
1253        temp &= ~IMOD_INTERVAL_MASK;
1254        temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1255        writel(temp, &pdev->ir_set->irq_control);
1256
1257        temp = readl(&pdev->port3x_regs->mode_addr);
1258
1259        switch (speed) {
1260        case USB_SPEED_SUPER_PLUS:
1261                temp |= CFG_3XPORT_SSP_SUPPORT;
1262                break;
1263        case USB_SPEED_SUPER:
1264                temp &= ~CFG_3XPORT_SSP_SUPPORT;
1265                break;
1266        case USB_SPEED_HIGH:
1267                break;
1268        case USB_SPEED_FULL:
1269                fs_speed = PORT_REG6_FORCE_FS;
1270                break;
1271        default:
1272                dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1273                        speed);
1274                fallthrough;
1275        case USB_SPEED_UNKNOWN:
1276                /* Default to superspeed. */
1277                speed = USB_SPEED_SUPER;
1278                break;
1279        }
1280
1281        if (speed >= USB_SPEED_SUPER) {
1282                writel(temp, &pdev->port3x_regs->mode_addr);
1283                cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1284                                     XDEV_RXDETECT);
1285        } else {
1286                cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1287        }
1288
1289        cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1290                             XDEV_RXDETECT);
1291
1292        cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1293
1294        writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1295
1296        ret = cdnsp_start(pdev);
1297        if (ret) {
1298                ret = -ENODEV;
1299                goto err;
1300        }
1301
1302        temp = readl(&pdev->op_regs->command);
1303        temp |= (CMD_INTE);
1304        writel(temp, &pdev->op_regs->command);
1305
1306        temp = readl(&pdev->ir_set->irq_pending);
1307        writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1308
1309        trace_cdnsp_init("Controller ready to work");
1310        return 0;
1311err:
1312        cdnsp_halt(pdev);
1313        return ret;
1314}
1315
1316static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1317                                  struct usb_gadget_driver *driver)
1318{
1319        enum usb_device_speed max_speed = driver->max_speed;
1320        struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1321        unsigned long flags;
1322        int ret;
1323
1324        spin_lock_irqsave(&pdev->lock, flags);
1325        pdev->gadget_driver = driver;
1326
1327        /* limit speed if necessary */
1328        max_speed = min(driver->max_speed, g->max_speed);
1329        ret = cdnsp_run(pdev, max_speed);
1330
1331        spin_unlock_irqrestore(&pdev->lock, flags);
1332
1333        return ret;
1334}
1335
1336/*
1337 * Update Event Ring Dequeue Pointer:
1338 * - When all events have finished
1339 * - To avoid "Event Ring Full Error" condition
1340 */
1341void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1342                               union cdnsp_trb *event_ring_deq,
1343                               u8 clear_ehb)
1344{
1345        u64 temp_64;
1346        dma_addr_t deq;
1347
1348        temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1349
1350        /* If necessary, update the HW's version of the event ring deq ptr. */
1351        if (event_ring_deq != pdev->event_ring->dequeue) {
1352                deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1353                                            pdev->event_ring->dequeue);
1354                temp_64 &= ERST_PTR_MASK;
1355                temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1356        }
1357
1358        /* Clear the event handler busy flag (RW1C). */
1359        if (clear_ehb)
1360                temp_64 |= ERST_EHB;
1361        else
1362                temp_64 &= ~ERST_EHB;
1363
1364        cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1365}
1366
1367static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1368{
1369        struct cdnsp_segment *seg;
1370        u64 val_64;
1371        int i;
1372
1373        cdnsp_initialize_ring_info(pdev->cmd_ring);
1374
1375        seg = pdev->cmd_ring->first_seg;
1376        for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1377                memset(seg->trbs, 0,
1378                       sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1379                seg = seg->next;
1380        }
1381
1382        /* Set the address in the Command Ring Control register. */
1383        val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1384        val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1385                 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1386                 pdev->cmd_ring->cycle_state;
1387        cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1388}
1389
1390static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1391{
1392        struct cdnsp_segment *event_deq_seg;
1393        union cdnsp_trb *event_ring_deq;
1394        union cdnsp_trb *event;
1395        u32 cycle_bit;
1396
1397        event_ring_deq = pdev->event_ring->dequeue;
1398        event_deq_seg = pdev->event_ring->deq_seg;
1399        event = pdev->event_ring->dequeue;
1400
1401        /* Update ring dequeue pointer. */
1402        while (1) {
1403                cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1404
1405                /* Does the controller or driver own the TRB? */
1406                if (cycle_bit != pdev->event_ring->cycle_state)
1407                        break;
1408
1409                cdnsp_inc_deq(pdev, pdev->event_ring);
1410
1411                if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1412                        event++;
1413                        continue;
1414                }
1415
1416                if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1417                                           event))
1418                        cycle_bit ^= 1;
1419
1420                event_deq_seg = event_deq_seg->next;
1421                event = event_deq_seg->trbs;
1422        }
1423
1424        cdnsp_update_erst_dequeue(pdev,  event_ring_deq, 1);
1425}
1426
1427static void cdnsp_stop(struct cdnsp_device *pdev)
1428{
1429        u32 temp;
1430
1431        cdnsp_cmd_flush_ep(pdev, &pdev->eps[0]);
1432
1433        /* Remove internally queued request for ep0. */
1434        if (!list_empty(&pdev->eps[0].pending_list)) {
1435                struct cdnsp_request *req;
1436
1437                req = next_request(&pdev->eps[0].pending_list);
1438                if (req == &pdev->ep0_preq)
1439                        cdnsp_ep_dequeue(&pdev->eps[0], req);
1440        }
1441
1442        cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1443        cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1444        cdnsp_disable_slot(pdev);
1445        cdnsp_halt(pdev);
1446
1447        temp = readl(&pdev->op_regs->status);
1448        writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1449        temp = readl(&pdev->ir_set->irq_pending);
1450        writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1451
1452        cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1453        cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1454
1455        /* Clear interrupt line */
1456        temp = readl(&pdev->ir_set->irq_pending);
1457        temp |= IMAN_IP;
1458        writel(temp, &pdev->ir_set->irq_pending);
1459
1460        cdnsp_consume_all_events(pdev);
1461        cdnsp_clear_cmd_ring(pdev);
1462
1463        trace_cdnsp_exit("Controller stopped.");
1464}
1465
1466/*
1467 * Stop controller.
1468 * This function is called by the gadget core when the driver is removed.
1469 * Disable slot, disable IRQs, and quiesce the controller.
1470 */
1471static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1472{
1473        struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1474        unsigned long flags;
1475
1476        spin_lock_irqsave(&pdev->lock, flags);
1477        cdnsp_stop(pdev);
1478        pdev->gadget_driver = NULL;
1479        spin_unlock_irqrestore(&pdev->lock, flags);
1480
1481        return 0;
1482}
1483
1484static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1485{
1486        struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1487
1488        return cdnsp_get_frame(pdev);
1489}
1490
1491static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1492{
1493        struct cdnsp_port_regs __iomem *port_regs;
1494        u32 portpm, portsc;
1495
1496        port_regs = pdev->active_port->regs;
1497        portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1498
1499        /* Remote wakeup feature is not enabled by host. */
1500        if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1501                portpm = readl(&port_regs->portpmsc);
1502
1503                if (!(portpm & PORT_RWE))
1504                        return;
1505        }
1506
1507        if (portsc == XDEV_U3 && !pdev->may_wakeup)
1508                return;
1509
1510        cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1511
1512        pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1513}
1514
1515static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1516{
1517        struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1518        unsigned long flags;
1519
1520        spin_lock_irqsave(&pdev->lock, flags);
1521        __cdnsp_gadget_wakeup(pdev);
1522        spin_unlock_irqrestore(&pdev->lock, flags);
1523
1524        return 0;
1525}
1526
1527static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1528                                        int is_selfpowered)
1529{
1530        struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1531        unsigned long flags;
1532
1533        spin_lock_irqsave(&pdev->lock, flags);
1534        g->is_selfpowered = !!is_selfpowered;
1535        spin_unlock_irqrestore(&pdev->lock, flags);
1536
1537        return 0;
1538}
1539
1540static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1541{
1542        struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1543        struct cdns *cdns = dev_get_drvdata(pdev->dev);
1544
1545        trace_cdnsp_pullup(is_on);
1546
1547        if (!is_on) {
1548                cdnsp_reset_device(pdev);
1549                cdns_clear_vbus(cdns);
1550        } else {
1551                cdns_set_vbus(cdns);
1552        }
1553        return 0;
1554}
1555
1556static const struct usb_gadget_ops cdnsp_gadget_ops = {
1557        .get_frame              = cdnsp_gadget_get_frame,
1558        .wakeup                 = cdnsp_gadget_wakeup,
1559        .set_selfpowered        = cdnsp_gadget_set_selfpowered,
1560        .pullup                 = cdnsp_gadget_pullup,
1561        .udc_start              = cdnsp_gadget_udc_start,
1562        .udc_stop               = cdnsp_gadget_udc_stop,
1563};
1564
1565static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1566                                   struct cdnsp_ep *pep)
1567{
1568        void __iomem *reg = &pdev->cap_regs->hc_capbase;
1569        int endpoints;
1570
1571        reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1572
1573        if (!pep->direction) {
1574                pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1575                pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1576                pep->buffering = (pep->buffering + 1) / 2;
1577                pep->buffering_period = (pep->buffering_period + 1) / 2;
1578                return;
1579        }
1580
1581        endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1582
1583        /* Set to XBUF_TX_TAG_MASK_0 register. */
1584        reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1585        /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1586        reg += pep->number * sizeof(u32) * 2;
1587
1588        pep->buffering = (readl(reg) + 1) / 2;
1589        pep->buffering_period = pep->buffering;
1590}
1591
1592static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1593{
1594        int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1595        struct cdnsp_ep *pep;
1596        int i;
1597
1598        INIT_LIST_HEAD(&pdev->gadget.ep_list);
1599
1600        if (max_streams < STREAM_LOG_STREAMS) {
1601                dev_err(pdev->dev, "Stream size %d not supported\n",
1602                        max_streams);
1603                return -EINVAL;
1604        }
1605
1606        max_streams = STREAM_LOG_STREAMS;
1607
1608        for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1609                bool direction = !(i & 1); /* Start from OUT endpoint. */
1610                u8 epnum = ((i + 1) >> 1);
1611
1612                if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1613                        continue;
1614
1615                pep = &pdev->eps[i];
1616                pep->pdev = pdev;
1617                pep->number = epnum;
1618                pep->direction = direction; /* 0 for OUT, 1 for IN. */
1619
1620                /*
1621                 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1622                 * pdev->eps[0]
1623                 */
1624                if (epnum == 0) {
1625                        snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1626                                 epnum, "BiDir");
1627
1628                        pep->idx = 0;
1629                        usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1630                        pep->endpoint.maxburst = 1;
1631                        pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1632                        pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1633                        pep->endpoint.comp_desc = NULL;
1634                        pep->endpoint.caps.type_control = true;
1635                        pep->endpoint.caps.dir_in = true;
1636                        pep->endpoint.caps.dir_out = true;
1637
1638                        pdev->ep0_preq.epnum = pep->number;
1639                        pdev->ep0_preq.pep = pep;
1640                        pdev->gadget.ep0 = &pep->endpoint;
1641                } else {
1642                        snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1643                                 epnum, (pep->direction) ? "in" : "out");
1644
1645                        pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;
1646                        usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1647
1648                        pep->endpoint.max_streams = max_streams;
1649                        pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1650                        list_add_tail(&pep->endpoint.ep_list,
1651                                      &pdev->gadget.ep_list);
1652
1653                        pep->endpoint.caps.type_iso = true;
1654                        pep->endpoint.caps.type_bulk = true;
1655                        pep->endpoint.caps.type_int = true;
1656
1657                        pep->endpoint.caps.dir_in = direction;
1658                        pep->endpoint.caps.dir_out = !direction;
1659                }
1660
1661                pep->endpoint.name = pep->name;
1662                pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1663                pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1664                cdnsp_get_ep_buffering(pdev, pep);
1665
1666                dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1667                        "CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1668                        "SupDir IN: %s, OUT: %s\n",
1669                        pep->name, 1024,
1670                        (pep->endpoint.caps.type_control) ? "yes" : "no",
1671                        (pep->endpoint.caps.type_int) ? "yes" : "no",
1672                        (pep->endpoint.caps.type_bulk) ? "yes" : "no",
1673                        (pep->endpoint.caps.type_iso) ? "yes" : "no",
1674                        (pep->endpoint.caps.dir_in) ? "yes" : "no",
1675                        (pep->endpoint.caps.dir_out) ? "yes" : "no");
1676
1677                INIT_LIST_HEAD(&pep->pending_list);
1678        }
1679
1680        return 0;
1681}
1682
1683static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1684{
1685        struct cdnsp_ep *pep;
1686        int i;
1687
1688        for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1689                pep = &pdev->eps[i];
1690                if (pep->number != 0 && pep->out_ctx)
1691                        list_del(&pep->endpoint.ep_list);
1692        }
1693}
1694
1695void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1696{
1697        pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1698
1699        if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1700                spin_unlock(&pdev->lock);
1701                pdev->gadget_driver->disconnect(&pdev->gadget);
1702                spin_lock(&pdev->lock);
1703        }
1704
1705        pdev->gadget.speed = USB_SPEED_UNKNOWN;
1706        usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1707
1708        pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1709}
1710
1711void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1712{
1713        if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1714                spin_unlock(&pdev->lock);
1715                pdev->gadget_driver->suspend(&pdev->gadget);
1716                spin_lock(&pdev->lock);
1717        }
1718}
1719
1720void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1721{
1722        if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1723                spin_unlock(&pdev->lock);
1724                pdev->gadget_driver->resume(&pdev->gadget);
1725                spin_lock(&pdev->lock);
1726        }
1727}
1728
1729void cdnsp_irq_reset(struct cdnsp_device *pdev)
1730{
1731        struct cdnsp_port_regs __iomem *port_regs;
1732
1733        cdnsp_reset_device(pdev);
1734
1735        port_regs = pdev->active_port->regs;
1736        pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1737
1738        spin_unlock(&pdev->lock);
1739        usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1740        spin_lock(&pdev->lock);
1741
1742        switch (pdev->gadget.speed) {
1743        case USB_SPEED_SUPER_PLUS:
1744        case USB_SPEED_SUPER:
1745                cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1746                pdev->gadget.ep0->maxpacket = 512;
1747                break;
1748        case USB_SPEED_HIGH:
1749        case USB_SPEED_FULL:
1750                cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1751                pdev->gadget.ep0->maxpacket = 64;
1752                break;
1753        default:
1754                /* Low speed is not supported. */
1755                dev_err(pdev->dev, "Unknown device speed\n");
1756                break;
1757        }
1758
1759        cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1760        cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1761        usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1762}
1763
1764static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1765{
1766        void __iomem *reg = &pdev->cap_regs->hc_capbase;
1767
1768        reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1769        pdev->rev_cap  = reg;
1770
1771        dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1772                 readl(&pdev->rev_cap->ctrl_revision),
1773                 readl(&pdev->rev_cap->rtl_revision),
1774                 readl(&pdev->rev_cap->ep_supported),
1775                 readl(&pdev->rev_cap->rx_buff_size),
1776                 readl(&pdev->rev_cap->tx_buff_size));
1777}
1778
1779static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1780{
1781        int ret;
1782        u32 reg;
1783
1784        pdev->cap_regs = pdev->regs;
1785        pdev->op_regs = pdev->regs +
1786                HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1787        pdev->run_regs = pdev->regs +
1788                (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1789
1790        /* Cache read-only capability registers */
1791        pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1792        pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1793        pdev->hci_version = HC_VERSION(pdev->hcc_params);
1794        pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1795
1796        cdnsp_get_rev_cap(pdev);
1797
1798        /* Make sure the Device Controller is halted. */
1799        ret = cdnsp_halt(pdev);
1800        if (ret)
1801                return ret;
1802
1803        /* Reset the internal controller memory state and registers. */
1804        ret = cdnsp_reset(pdev);
1805        if (ret)
1806                return ret;
1807
1808        /*
1809         * Set dma_mask and coherent_dma_mask to 64-bits,
1810         * if controller supports 64-bit addressing.
1811         */
1812        if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1813            !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1814                dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1815                dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1816        } else {
1817                /*
1818                 * This is to avoid error in cases where a 32-bit USB
1819                 * controller is used on a 64-bit capable system.
1820                 */
1821                ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1822                if (ret)
1823                        return ret;
1824
1825                dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1826                dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1827        }
1828
1829        spin_lock_init(&pdev->lock);
1830
1831        ret = cdnsp_mem_init(pdev);
1832        if (ret)
1833                return ret;
1834
1835        /*
1836         * Software workaround for U1: after transition
1837         * to U1 the controller starts gating clock, and in some cases,
1838         * it causes that controller stack.
1839         */
1840        reg = readl(&pdev->port3x_regs->mode_2);
1841        reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1842        writel(reg, &pdev->port3x_regs->mode_2);
1843
1844        return 0;
1845}
1846
1847static int __cdnsp_gadget_init(struct cdns *cdns)
1848{
1849        struct cdnsp_device *pdev;
1850        u32 max_speed;
1851        int ret = -ENOMEM;
1852
1853        cdns_drd_gadget_on(cdns);
1854
1855        pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1856        if (!pdev)
1857                return -ENOMEM;
1858
1859        pm_runtime_get_sync(cdns->dev);
1860
1861        cdns->gadget_dev = pdev;
1862        pdev->dev = cdns->dev;
1863        pdev->regs = cdns->dev_regs;
1864        max_speed = usb_get_maximum_speed(cdns->dev);
1865
1866        switch (max_speed) {
1867        case USB_SPEED_FULL:
1868        case USB_SPEED_HIGH:
1869        case USB_SPEED_SUPER:
1870        case USB_SPEED_SUPER_PLUS:
1871                break;
1872        default:
1873                dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1874                fallthrough;
1875        case USB_SPEED_UNKNOWN:
1876                /* Default to SSP */
1877                max_speed = USB_SPEED_SUPER_PLUS;
1878                break;
1879        }
1880
1881        pdev->gadget.ops = &cdnsp_gadget_ops;
1882        pdev->gadget.name = "cdnsp-gadget";
1883        pdev->gadget.speed = USB_SPEED_UNKNOWN;
1884        pdev->gadget.sg_supported = 1;
1885        pdev->gadget.max_speed = max_speed;
1886        pdev->gadget.lpm_capable = 1;
1887
1888        pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1889        if (!pdev->setup_buf)
1890                goto free_pdev;
1891
1892        /*
1893         * Controller supports not aligned buffer but it should improve
1894         * performance.
1895         */
1896        pdev->gadget.quirk_ep_out_aligned_size = true;
1897
1898        ret = cdnsp_gen_setup(pdev);
1899        if (ret) {
1900                dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1901                goto free_setup;
1902        }
1903
1904        ret = cdnsp_gadget_init_endpoints(pdev);
1905        if (ret) {
1906                dev_err(pdev->dev, "failed to initialize endpoints\n");
1907                goto halt_pdev;
1908        }
1909
1910        ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1911        if (ret) {
1912                dev_err(pdev->dev, "failed to register udc\n");
1913                goto free_endpoints;
1914        }
1915
1916        ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1917                                        cdnsp_irq_handler,
1918                                        cdnsp_thread_irq_handler, IRQF_SHARED,
1919                                        dev_name(pdev->dev), pdev);
1920        if (ret)
1921                goto del_gadget;
1922
1923        return 0;
1924
1925del_gadget:
1926        usb_del_gadget_udc(&pdev->gadget);
1927free_endpoints:
1928        cdnsp_gadget_free_endpoints(pdev);
1929halt_pdev:
1930        cdnsp_halt(pdev);
1931        cdnsp_reset(pdev);
1932        cdnsp_mem_cleanup(pdev);
1933free_setup:
1934        kfree(pdev->setup_buf);
1935free_pdev:
1936        kfree(pdev);
1937
1938        return ret;
1939}
1940
1941static void cdnsp_gadget_exit(struct cdns *cdns)
1942{
1943        struct cdnsp_device *pdev = cdns->gadget_dev;
1944
1945        devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1946        pm_runtime_mark_last_busy(cdns->dev);
1947        pm_runtime_put_autosuspend(cdns->dev);
1948        usb_del_gadget_udc(&pdev->gadget);
1949        cdnsp_gadget_free_endpoints(pdev);
1950        cdnsp_mem_cleanup(pdev);
1951        kfree(pdev);
1952        cdns->gadget_dev = NULL;
1953        cdns_drd_gadget_off(cdns);
1954}
1955
1956static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
1957{
1958        struct cdnsp_device *pdev = cdns->gadget_dev;
1959        unsigned long flags;
1960
1961        if (pdev->link_state == XDEV_U3)
1962                return 0;
1963
1964        spin_lock_irqsave(&pdev->lock, flags);
1965        cdnsp_disconnect_gadget(pdev);
1966        cdnsp_stop(pdev);
1967        spin_unlock_irqrestore(&pdev->lock, flags);
1968
1969        return 0;
1970}
1971
1972static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated)
1973{
1974        struct cdnsp_device *pdev = cdns->gadget_dev;
1975        enum usb_device_speed max_speed;
1976        unsigned long flags;
1977        int ret;
1978
1979        if (!pdev->gadget_driver)
1980                return 0;
1981
1982        spin_lock_irqsave(&pdev->lock, flags);
1983        max_speed = pdev->gadget_driver->max_speed;
1984
1985        /* Limit speed if necessary. */
1986        max_speed = min(max_speed, pdev->gadget.max_speed);
1987
1988        ret = cdnsp_run(pdev, max_speed);
1989
1990        if (pdev->link_state == XDEV_U3)
1991                __cdnsp_gadget_wakeup(pdev);
1992
1993        spin_unlock_irqrestore(&pdev->lock, flags);
1994
1995        return ret;
1996}
1997
1998/**
1999 * cdnsp_gadget_init - initialize device structure
2000 * @cdns: cdnsp instance
2001 *
2002 * This function initializes the gadget.
2003 */
2004int cdnsp_gadget_init(struct cdns *cdns)
2005{
2006        struct cdns_role_driver *rdrv;
2007
2008        rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2009        if (!rdrv)
2010                return -ENOMEM;
2011
2012        rdrv->start     = __cdnsp_gadget_init;
2013        rdrv->stop      = cdnsp_gadget_exit;
2014        rdrv->suspend   = cdnsp_gadget_suspend;
2015        rdrv->resume    = cdnsp_gadget_resume;
2016        rdrv->state     = CDNS_ROLE_STATE_INACTIVE;
2017        rdrv->name      = "gadget";
2018        cdns->roles[USB_ROLE_DEVICE] = rdrv;
2019
2020        return 0;
2021}
2022