/*
 * Intel Langwell USB Device Controller driver
 * Copyright (C) 2008-2009, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 */


/* #undef       DEBUG */
/* #undef       VERBOSE_DEBUG */

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/ioport.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/timer.h>
#include <linux/list.h>
#include <linux/interrupt.h>
#include <linux/moduleparam.h>
#include <linux/device.h>
#include <linux/usb/ch9.h>
#include <linux/usb/gadget.h>
#include <linux/usb/otg.h>
#include <linux/pm.h>
#include <linux/io.h>
#include <linux/irq.h>
#include <asm/system.h>
#include <asm/unaligned.h>

#include "langwell_udc.h"


#define DRIVER_DESC             "Intel Langwell USB Device Controller driver"
#define DRIVER_VERSION          "16 May 2009"

static const char driver_name[] = "langwell_udc";
static const char driver_desc[] = DRIVER_DESC;


/* for endpoint 0 operations */
static const struct usb_endpoint_descriptor
langwell_ep0_desc = {
        .bLength =              USB_DT_ENDPOINT_SIZE,
        .bDescriptorType =      USB_DT_ENDPOINT,
        .bEndpointAddress =     0,
        .bmAttributes =         USB_ENDPOINT_XFER_CONTROL,
        .wMaxPacketSize =       EP0_MAX_PKT_SIZE,
};


/*-------------------------------------------------------------------------*/
/* debugging */

#ifdef  VERBOSE_DEBUG
static inline void print_all_registers(struct langwell_udc *dev)
{
        int     i;

        /* Capability Registers */
        dev_dbg(&dev->pdev->dev,
                "Capability Registers (offset: 0x%04x, length: 0x%08x)\n",
                CAP_REG_OFFSET, (u32)sizeof(struct langwell_cap_regs));
        dev_dbg(&dev->pdev->dev, "caplength=0x%02x\n",
                        readb(&dev->cap_regs->caplength));
        dev_dbg(&dev->pdev->dev, "hciversion=0x%04x\n",
                        readw(&dev->cap_regs->hciversion));
        dev_dbg(&dev->pdev->dev, "hcsparams=0x%08x\n",
                        readl(&dev->cap_regs->hcsparams));
        dev_dbg(&dev->pdev->dev, "hccparams=0x%08x\n",
                        readl(&dev->cap_regs->hccparams));
        dev_dbg(&dev->pdev->dev, "dciversion=0x%04x\n",
                        readw(&dev->cap_regs->dciversion));
        dev_dbg(&dev->pdev->dev, "dccparams=0x%08x\n",
                        readl(&dev->cap_regs->dccparams));

        /* Operational Registers */
        dev_dbg(&dev->pdev->dev,
                "Operational Registers (offset: 0x%04x, length: 0x%08x)\n",
                OP_REG_OFFSET, (u32)sizeof(struct langwell_op_regs));
        dev_dbg(&dev->pdev->dev, "extsts=0x%08x\n",
                        readl(&dev->op_regs->extsts));
        dev_dbg(&dev->pdev->dev, "extintr=0x%08x\n",
                        readl(&dev->op_regs->extintr));
        dev_dbg(&dev->pdev->dev, "usbcmd=0x%08x\n",
                        readl(&dev->op_regs->usbcmd));
        dev_dbg(&dev->pdev->dev, "usbsts=0x%08x\n",
                        readl(&dev->op_regs->usbsts));
        dev_dbg(&dev->pdev->dev, "usbintr=0x%08x\n",
                        readl(&dev->op_regs->usbintr));
        dev_dbg(&dev->pdev->dev, "frindex=0x%08x\n",
                        readl(&dev->op_regs->frindex));
        dev_dbg(&dev->pdev->dev, "ctrldssegment=0x%08x\n",
                        readl(&dev->op_regs->ctrldssegment));
        dev_dbg(&dev->pdev->dev, "deviceaddr=0x%08x\n",
                        readl(&dev->op_regs->deviceaddr));
        dev_dbg(&dev->pdev->dev, "endpointlistaddr=0x%08x\n",
                        readl(&dev->op_regs->endpointlistaddr));
        dev_dbg(&dev->pdev->dev, "ttctrl=0x%08x\n",
                        readl(&dev->op_regs->ttctrl));
        dev_dbg(&dev->pdev->dev, "burstsize=0x%08x\n",
                        readl(&dev->op_regs->burstsize));
        dev_dbg(&dev->pdev->dev, "txfilltuning=0x%08x\n",
                        readl(&dev->op_regs->txfilltuning));
        dev_dbg(&dev->pdev->dev, "txttfilltuning=0x%08x\n",
                        readl(&dev->op_regs->txttfilltuning));
        dev_dbg(&dev->pdev->dev, "ic_usb=0x%08x\n",
                        readl(&dev->op_regs->ic_usb));
        dev_dbg(&dev->pdev->dev, "ulpi_viewport=0x%08x\n",
                        readl(&dev->op_regs->ulpi_viewport));
        dev_dbg(&dev->pdev->dev, "configflag=0x%08x\n",
                        readl(&dev->op_regs->configflag));
        dev_dbg(&dev->pdev->dev, "portsc1=0x%08x\n",
                        readl(&dev->op_regs->portsc1));
        dev_dbg(&dev->pdev->dev, "devlc=0x%08x\n",
                        readl(&dev->op_regs->devlc));
        dev_dbg(&dev->pdev->dev, "otgsc=0x%08x\n",
                        readl(&dev->op_regs->otgsc));
        dev_dbg(&dev->pdev->dev, "usbmode=0x%08x\n",
                        readl(&dev->op_regs->usbmode));
        dev_dbg(&dev->pdev->dev, "endptnak=0x%08x\n",
                        readl(&dev->op_regs->endptnak));
        dev_dbg(&dev->pdev->dev, "endptnaken=0x%08x\n",
                        readl(&dev->op_regs->endptnaken));
        dev_dbg(&dev->pdev->dev, "endptsetupstat=0x%08x\n",
                        readl(&dev->op_regs->endptsetupstat));
        dev_dbg(&dev->pdev->dev, "endptprime=0x%08x\n",
                        readl(&dev->op_regs->endptprime));
        dev_dbg(&dev->pdev->dev, "endptflush=0x%08x\n",
                        readl(&dev->op_regs->endptflush));
        dev_dbg(&dev->pdev->dev, "endptstat=0x%08x\n",
                        readl(&dev->op_regs->endptstat));
        dev_dbg(&dev->pdev->dev, "endptcomplete=0x%08x\n",
                        readl(&dev->op_regs->endptcomplete));

        for (i = 0; i < dev->ep_max / 2; i++) {
                dev_dbg(&dev->pdev->dev, "endptctrl[%d]=0x%08x\n",
                                i, readl(&dev->op_regs->endptctrl[i]));
        }
}
#else

#define print_all_registers(dev)        do { } while (0)

#endif /* VERBOSE_DEBUG */


/*-------------------------------------------------------------------------*/

#define is_in(ep)       (((ep)->ep_num == 0) ? ((ep)->dev->ep0_dir ==   \
                        USB_DIR_IN) : (usb_endpoint_dir_in((ep)->desc)))

#define DIR_STRING(ep)  (is_in(ep) ? "in" : "out")


static char *type_string(const struct usb_endpoint_descriptor *desc)
{
        switch (usb_endpoint_type(desc)) {
        case USB_ENDPOINT_XFER_BULK:
                return "bulk";
        case USB_ENDPOINT_XFER_ISOC:
                return "iso";
        case USB_ENDPOINT_XFER_INT:
                return "int";
        };

        return "control";
}


/* configure endpoint control registers */
static void ep_reset(struct langwell_ep *ep, unsigned char ep_num,
                unsigned char is_in, unsigned char ep_type)
{
        struct langwell_udc     *dev;
        u32                     endptctrl;

        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        endptctrl = readl(&dev->op_regs->endptctrl[ep_num]);
        if (is_in) {    /* TX */
                if (ep_num)
                        endptctrl |= EPCTRL_TXR;
                endptctrl |= EPCTRL_TXE;
                endptctrl |= ep_type << EPCTRL_TXT_SHIFT;
        } else {        /* RX */
                if (ep_num)
                        endptctrl |= EPCTRL_RXR;
                endptctrl |= EPCTRL_RXE;
                endptctrl |= ep_type << EPCTRL_RXT_SHIFT;
        }

        writel(endptctrl, &dev->op_regs->endptctrl[ep_num]);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/* reset ep0 dQH and endptctrl */
static void ep0_reset(struct langwell_udc *dev)
{
        struct langwell_ep      *ep;
        int                     i;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* ep0 in and out */
        for (i = 0; i < 2; i++) {
                ep = &dev->ep[i];
                ep->dev = dev;

                /* ep0 dQH */
                ep->dqh = &dev->ep_dqh[i];

                /* configure ep0 endpoint capabilities in dQH */
                ep->dqh->dqh_ios = 1;
                ep->dqh->dqh_mpl = EP0_MAX_PKT_SIZE;

                /* enable ep0-in HW zero length termination select */
                if (is_in(ep))
                        ep->dqh->dqh_zlt = 0;
                ep->dqh->dqh_mult = 0;

                ep->dqh->dtd_next = DTD_TERM;

                /* configure ep0 control registers */
                ep_reset(&dev->ep[0], 0, i, USB_ENDPOINT_XFER_CONTROL);
        }

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/*-------------------------------------------------------------------------*/

/* endpoints operations */

/* configure endpoint, making it usable */
static int langwell_ep_enable(struct usb_ep *_ep,
                const struct usb_endpoint_descriptor *desc)
{
        struct langwell_udc     *dev;
        struct langwell_ep      *ep;
        u16                     max = 0;
        unsigned long           flags;
        int                     i, retval = 0;
        unsigned char           zlt, ios = 0, mult = 0;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !desc || ep->desc
                        || desc->bDescriptorType != USB_DT_ENDPOINT)
                return -EINVAL;

        if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
                return -ESHUTDOWN;

        max = usb_endpoint_maxp(desc);

        /*
         * disable HW zero length termination select
         * driver handles zero length packet through req->req.zero
         */
        zlt = 1;

        /*
         * sanity check type, direction, address, and then
         * initialize the endpoint capabilities fields in dQH
         */
        switch (usb_endpoint_type(desc)) {
        case USB_ENDPOINT_XFER_CONTROL:
                ios = 1;
                break;
        case USB_ENDPOINT_XFER_BULK:
                if ((dev->gadget.speed == USB_SPEED_HIGH
                                        && max != 512)
                                || (dev->gadget.speed == USB_SPEED_FULL
                                        && max > 64)) {
                        goto done;
                }
                break;
        case USB_ENDPOINT_XFER_INT:
                if (strstr(ep->ep.name, "-iso")) /* bulk is ok */
                        goto done;

                switch (dev->gadget.speed) {
                case USB_SPEED_HIGH:
                        if (max <= 1024)
                                break;
                case USB_SPEED_FULL:
                        if (max <= 64)
                                break;
                default:
                        if (max <= 8)
                                break;
                        goto done;
                }
                break;
        case USB_ENDPOINT_XFER_ISOC:
                if (strstr(ep->ep.name, "-bulk")
                                || strstr(ep->ep.name, "-int"))
                        goto done;

                switch (dev->gadget.speed) {
                case USB_SPEED_HIGH:
                        if (max <= 1024)
                                break;
                case USB_SPEED_FULL:
                        if (max <= 1023)
                                break;
                default:
                        goto done;
                }
                /*
                 * FIXME:
                 * calculate transactions needed for high bandwidth iso
                 */
                mult = (unsigned char)(1 + ((max >> 11) & 0x03));
                max = max & 0x8ff;      /* bit 0~10 */
                /* 3 transactions at most */
                if (mult > 3)
                        goto done;
                break;
        default:
                goto done;
        }

        spin_lock_irqsave(&dev->lock, flags);

        ep->ep.maxpacket = max;
        ep->desc = desc;
        ep->stopped = 0;
        ep->ep_num = usb_endpoint_num(desc);

        /* ep_type */
        ep->ep_type = usb_endpoint_type(desc);

        /* configure endpoint control registers */
        ep_reset(ep, ep->ep_num, is_in(ep), ep->ep_type);

        /* configure endpoint capabilities in dQH */
        i = ep->ep_num * 2 + is_in(ep);
        ep->dqh = &dev->ep_dqh[i];
        ep->dqh->dqh_ios = ios;
        ep->dqh->dqh_mpl = cpu_to_le16(max);
        ep->dqh->dqh_zlt = zlt;
        ep->dqh->dqh_mult = mult;
        ep->dqh->dtd_next = DTD_TERM;

        dev_dbg(&dev->pdev->dev, "enabled %s (ep%d%s-%s), max %04x\n",
                        _ep->name,
                        ep->ep_num,
                        DIR_STRING(ep),
                        type_string(desc),
                        max);

        spin_unlock_irqrestore(&dev->lock, flags);
done:
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return retval;
}


/*-------------------------------------------------------------------------*/

/* retire a request */
static void done(struct langwell_ep *ep, struct langwell_request *req,
                int status)
{
        struct langwell_udc     *dev = ep->dev;
        unsigned                stopped = ep->stopped;
        struct langwell_dtd     *curr_dtd, *next_dtd;
        int                     i;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* remove the req from ep->queue */
        list_del_init(&req->queue);

        if (req->req.status == -EINPROGRESS)
                req->req.status = status;
        else
                status = req->req.status;

        /* free dTD for the request */
        next_dtd = req->head;
        for (i = 0; i < req->dtd_count; i++) {
                curr_dtd = next_dtd;
                if (i != req->dtd_count - 1)
                        next_dtd = curr_dtd->next_dtd_virt;
                dma_pool_free(dev->dtd_pool, curr_dtd, curr_dtd->dtd_dma);
        }

        if (req->mapped) {
                dma_unmap_single(&dev->pdev->dev,
                        req->req.dma, req->req.length,
                        is_in(ep) ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);
                req->req.dma = DMA_ADDR_INVALID;
                req->mapped = 0;
        } else
                dma_sync_single_for_cpu(&dev->pdev->dev, req->req.dma,
                                req->req.length,
                                is_in(ep) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);

        if (status != -ESHUTDOWN)
                dev_dbg(&dev->pdev->dev,
                                "complete %s, req %p, stat %d, len %u/%u\n",
                                ep->ep.name, &req->req, status,
                                req->req.actual, req->req.length);

        /* don't modify queue heads during completion callback */
        ep->stopped = 1;

        spin_unlock(&dev->lock);
        /* complete routine from gadget driver */
        if (req->req.complete)
                req->req.complete(&ep->ep, &req->req);

        spin_lock(&dev->lock);
        ep->stopped = stopped;

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


static void langwell_ep_fifo_flush(struct usb_ep *_ep);

/* delete all endpoint requests, called with spinlock held */
static void nuke(struct langwell_ep *ep, int status)
{
        /* called with spinlock held */
        ep->stopped = 1;

        /* endpoint fifo flush */
        if (&ep->ep && ep->desc)
                langwell_ep_fifo_flush(&ep->ep);

        while (!list_empty(&ep->queue)) {
                struct langwell_request *req = NULL;
                req = list_entry(ep->queue.next, struct langwell_request,
                                queue);
                done(ep, req, status);
        }
}


/*-------------------------------------------------------------------------*/

/* endpoint is no longer usable */
static int langwell_ep_disable(struct usb_ep *_ep)
{
        struct langwell_ep      *ep;
        unsigned long           flags;
        struct langwell_udc     *dev;
        int                     ep_num;
        u32                     endptctrl;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !ep->desc)
                return -EINVAL;

        spin_lock_irqsave(&dev->lock, flags);

        /* disable endpoint control register */
        ep_num = ep->ep_num;
        endptctrl = readl(&dev->op_regs->endptctrl[ep_num]);
        if (is_in(ep))
                endptctrl &= ~EPCTRL_TXE;
        else
                endptctrl &= ~EPCTRL_RXE;
        writel(endptctrl, &dev->op_regs->endptctrl[ep_num]);

        /* nuke all pending requests (does flush) */
        nuke(ep, -ESHUTDOWN);

        ep->desc = NULL;
        ep->ep.desc = NULL;
        ep->stopped = 1;

        spin_unlock_irqrestore(&dev->lock, flags);

        dev_dbg(&dev->pdev->dev, "disabled %s\n", _ep->name);
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);

        return 0;
}


/* allocate a request object to use with this endpoint */
static struct usb_request *langwell_alloc_request(struct usb_ep *_ep,
                gfp_t gfp_flags)
{
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;
        struct langwell_request *req = NULL;

        if (!_ep)
                return NULL;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        req = kzalloc(sizeof(*req), gfp_flags);
        if (!req)
                return NULL;

        req->req.dma = DMA_ADDR_INVALID;
        INIT_LIST_HEAD(&req->queue);

        dev_vdbg(&dev->pdev->dev, "alloc request for %s\n", _ep->name);
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return &req->req;
}


/* free a request object */
static void langwell_free_request(struct usb_ep *_ep,
                struct usb_request *_req)
{
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;
        struct langwell_request *req = NULL;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !_req)
                return;

        req = container_of(_req, struct langwell_request, req);
        WARN_ON(!list_empty(&req->queue));

        if (_req)
                kfree(req);

        dev_vdbg(&dev->pdev->dev, "free request for %s\n", _ep->name);
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/*-------------------------------------------------------------------------*/

/* queue dTD and PRIME endpoint */
static int queue_dtd(struct langwell_ep *ep, struct langwell_request *req)
{
        u32                     bit_mask, usbcmd, endptstat, dtd_dma;
        u8                      dtd_status;
        int                     i;
        struct langwell_dqh     *dqh;
        struct langwell_udc     *dev;

        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        i = ep->ep_num * 2 + is_in(ep);
        dqh = &dev->ep_dqh[i];

        if (ep->ep_num)
                dev_vdbg(&dev->pdev->dev, "%s\n", ep->name);
        else
                /* ep0 */
                dev_vdbg(&dev->pdev->dev, "%s-%s\n", ep->name, DIR_STRING(ep));

        dev_vdbg(&dev->pdev->dev, "ep_dqh[%d] addr: 0x%p\n",
                        i, &(dev->ep_dqh[i]));

        bit_mask = is_in(ep) ?
                (1 << (ep->ep_num + 16)) : (1 << (ep->ep_num));

        dev_vdbg(&dev->pdev->dev, "bit_mask = 0x%08x\n", bit_mask);

        /* check if the pipe is empty */
        if (!(list_empty(&ep->queue))) {
                /* add dTD to the end of linked list */
                struct langwell_request *lastreq;
                lastreq = list_entry(ep->queue.prev,
                                struct langwell_request, queue);

                lastreq->tail->dtd_next =
                        cpu_to_le32(req->head->dtd_dma & DTD_NEXT_MASK);

                /* read prime bit, if 1 goto out */
                if (readl(&dev->op_regs->endptprime) & bit_mask)
                        goto out;

                do {
                        /* set ATDTW bit in USBCMD */
                        usbcmd = readl(&dev->op_regs->usbcmd);
                        writel(usbcmd | CMD_ATDTW, &dev->op_regs->usbcmd);

                        /* read correct status bit */
                        endptstat = readl(&dev->op_regs->endptstat) & bit_mask;

                } while (!(readl(&dev->op_regs->usbcmd) & CMD_ATDTW));

                /* write ATDTW bit to 0 */
                usbcmd = readl(&dev->op_regs->usbcmd);
                writel(usbcmd & ~CMD_ATDTW, &dev->op_regs->usbcmd);

                if (endptstat)
                        goto out;
        }

        /* write dQH next pointer and terminate bit to 0 */
        dtd_dma = req->head->dtd_dma & DTD_NEXT_MASK;
        dqh->dtd_next = cpu_to_le32(dtd_dma);

        /* clear active and halt bit */
        dtd_status = (u8) ~(DTD_STS_ACTIVE | DTD_STS_HALTED);
        dqh->dtd_status &= dtd_status;
        dev_vdbg(&dev->pdev->dev, "dqh->dtd_status = 0x%x\n", dqh->dtd_status);

        /* ensure that updates to the dQH will occur before priming */
        wmb();

        /* write 1 to endptprime register to PRIME endpoint */
        bit_mask = is_in(ep) ? (1 << (ep->ep_num + 16)) : (1 << ep->ep_num);
        dev_vdbg(&dev->pdev->dev, "endprime bit_mask = 0x%08x\n", bit_mask);
        writel(bit_mask, &dev->op_regs->endptprime);
out:
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}


/* fill in the dTD structure to build a transfer descriptor */
static struct langwell_dtd *build_dtd(struct langwell_request *req,
                unsigned *length, dma_addr_t *dma, int *is_last)
{
        u32                      buf_ptr;
        struct langwell_dtd     *dtd;
        struct langwell_udc     *dev;
        int                     i;

        dev = req->ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* the maximum transfer length, up to 16k bytes */
        *length = min(req->req.length - req->req.actual,
                        (unsigned)DTD_MAX_TRANSFER_LENGTH);

        /* create dTD dma_pool resource */
        dtd = dma_pool_alloc(dev->dtd_pool, GFP_KERNEL, dma);
        if (dtd == NULL)
                return dtd;
        dtd->dtd_dma = *dma;

        /* initialize buffer page pointers */
        buf_ptr = (u32)(req->req.dma + req->req.actual);
        for (i = 0; i < 5; i++)
                dtd->dtd_buf[i] = cpu_to_le32(buf_ptr + i * PAGE_SIZE);

        req->req.actual += *length;

        /* fill in total bytes with transfer size */
        dtd->dtd_total = cpu_to_le16(*length);
        dev_vdbg(&dev->pdev->dev, "dtd->dtd_total = %d\n", dtd->dtd_total);

        /* set is_last flag if req->req.zero is set or not */
        if (req->req.zero) {
                if (*length == 0 || (*length % req->ep->ep.maxpacket) != 0)
                        *is_last = 1;
                else
                        *is_last = 0;
        } else if (req->req.length == req->req.actual) {
                *is_last = 1;
        } else
                *is_last = 0;

        if (*is_last == 0)
                dev_vdbg(&dev->pdev->dev, "multi-dtd request!\n");

        /* set interrupt on complete bit for the last dTD */
        if (*is_last && !req->req.no_interrupt)
                dtd->dtd_ioc = 1;

        /* set multiplier override 0 for non-ISO and non-TX endpoint */
        dtd->dtd_multo = 0;

        /* set the active bit of status field to 1 */
        dtd->dtd_status = DTD_STS_ACTIVE;
        dev_vdbg(&dev->pdev->dev, "dtd->dtd_status = 0x%02x\n",
                        dtd->dtd_status);

        dev_vdbg(&dev->pdev->dev, "length = %d, dma addr= 0x%08x\n",
                        *length, (int)*dma);
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return dtd;
}


/* generate dTD linked list for a request */
static int req_to_dtd(struct langwell_request *req)
{
        unsigned                count;
        int                     is_last, is_first = 1;
        struct langwell_dtd     *dtd, *last_dtd = NULL;
        struct langwell_udc     *dev;
        dma_addr_t              dma;

        dev = req->ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);
        do {
                dtd = build_dtd(req, &count, &dma, &is_last);
                if (dtd == NULL)
                        return -ENOMEM;

                if (is_first) {
                        is_first = 0;
                        req->head = dtd;
                } else {
                        last_dtd->dtd_next = cpu_to_le32(dma);
                        last_dtd->next_dtd_virt = dtd;
                }
                last_dtd = dtd;
                req->dtd_count++;
        } while (!is_last);

        /* set terminate bit to 1 for the last dTD */
        dtd->dtd_next = DTD_TERM;

        req->tail = dtd;

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}

/*-------------------------------------------------------------------------*/

/* queue (submits) an I/O requests to an endpoint */
static int langwell_ep_queue(struct usb_ep *_ep, struct usb_request *_req,
                gfp_t gfp_flags)
{
        struct langwell_request *req;
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;
        unsigned long           flags;
        int                     is_iso = 0, zlflag = 0;

        /* always require a cpu-view buffer */
        req = container_of(_req, struct langwell_request, req);
        ep = container_of(_ep, struct langwell_ep, ep);

        if (!_req || !_req->complete || !_req->buf
                        || !list_empty(&req->queue)) {
                return -EINVAL;
        }

        if (unlikely(!_ep || !ep->desc))
                return -EINVAL;

        dev = ep->dev;
        req->ep = ep;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (usb_endpoint_xfer_isoc(ep->desc)) {
                if (req->req.length > ep->ep.maxpacket)
                        return -EMSGSIZE;
                is_iso = 1;
        }

        if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN))
                return -ESHUTDOWN;

        /* set up dma mapping in case the caller didn't */
        if (_req->dma == DMA_ADDR_INVALID) {
                /* WORKAROUND: WARN_ON(size == 0) */
                if (_req->length == 0) {
                        dev_vdbg(&dev->pdev->dev, "req->length: 0->1\n");
                        zlflag = 1;
                        _req->length++;
                }

                _req->dma = dma_map_single(&dev->pdev->dev,
                                _req->buf, _req->length,
                                is_in(ep) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
                if (zlflag && (_req->length == 1)) {
                        dev_vdbg(&dev->pdev->dev, "req->length: 1->0\n");
                        zlflag = 0;
                        _req->length = 0;
                }

                req->mapped = 1;
                dev_vdbg(&dev->pdev->dev, "req->mapped = 1\n");
        } else {
                dma_sync_single_for_device(&dev->pdev->dev,
                                _req->dma, _req->length,
                                is_in(ep) ?  DMA_TO_DEVICE : DMA_FROM_DEVICE);
                req->mapped = 0;
                dev_vdbg(&dev->pdev->dev, "req->mapped = 0\n");
        }

        dev_dbg(&dev->pdev->dev,
                        "%s queue req %p, len %u, buf %p, dma 0x%08x\n",
                        _ep->name,
                        _req, _req->length, _req->buf, (int)_req->dma);

        _req->status = -EINPROGRESS;
        _req->actual = 0;
        req->dtd_count = 0;

        spin_lock_irqsave(&dev->lock, flags);

        /* build and put dTDs to endpoint queue */
        if (!req_to_dtd(req)) {
                queue_dtd(ep, req);
        } else {
                spin_unlock_irqrestore(&dev->lock, flags);
                return -ENOMEM;
        }

        /* update ep0 state */
        if (ep->ep_num == 0)
                dev->ep0_state = DATA_STATE_XMIT;

        if (likely(req != NULL)) {
                list_add_tail(&req->queue, &ep->queue);
                dev_vdbg(&dev->pdev->dev, "list_add_tail()\n");
        }

        spin_unlock_irqrestore(&dev->lock, flags);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}


/* dequeue (cancels, unlinks) an I/O request from an endpoint */
static int langwell_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
{
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;
        struct langwell_request *req;
        unsigned long           flags;
        int                     stopped, ep_num, retval = 0;
        u32                     endptctrl;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !ep->desc || !_req)
                return -EINVAL;

        if (!dev->driver)
                return -ESHUTDOWN;

        spin_lock_irqsave(&dev->lock, flags);
        stopped = ep->stopped;

        /* quiesce dma while we patch the queue */
        ep->stopped = 1;
        ep_num = ep->ep_num;

        /* disable endpoint control register */
        endptctrl = readl(&dev->op_regs->endptctrl[ep_num]);
        if (is_in(ep))
                endptctrl &= ~EPCTRL_TXE;
        else
                endptctrl &= ~EPCTRL_RXE;
        writel(endptctrl, &dev->op_regs->endptctrl[ep_num]);

        /* make sure it's still queued on this endpoint */
        list_for_each_entry(req, &ep->queue, queue) {
                if (&req->req == _req)
                        break;
        }

        if (&req->req != _req) {
                retval = -EINVAL;
                goto done;
        }

        /* queue head may be partially complete. */
        if (ep->queue.next == &req->queue) {
                dev_dbg(&dev->pdev->dev, "unlink (%s) dma\n", _ep->name);
                _req->status = -ECONNRESET;
                langwell_ep_fifo_flush(&ep->ep);

                /* not the last request in endpoint queue */
                if (likely(ep->queue.next == &req->queue)) {
                        struct langwell_dqh     *dqh;
                        struct langwell_request *next_req;

                        dqh = ep->dqh;
                        next_req = list_entry(req->queue.next,
                                        struct langwell_request, queue);

                        /* point the dQH to the first dTD of next request */
                        writel((u32) next_req->head, &dqh->dqh_current);
                }
        } else {
                struct langwell_request *prev_req;

                prev_req = list_entry(req->queue.prev,
                                struct langwell_request, queue);
                writel(readl(&req->tail->dtd_next),
                                &prev_req->tail->dtd_next);
        }

        done(ep, req, -ECONNRESET);

done:
        /* enable endpoint again */
        endptctrl = readl(&dev->op_regs->endptctrl[ep_num]);
        if (is_in(ep))
                endptctrl |= EPCTRL_TXE;
        else
                endptctrl |= EPCTRL_RXE;
        writel(endptctrl, &dev->op_regs->endptctrl[ep_num]);

        ep->stopped = stopped;
        spin_unlock_irqrestore(&dev->lock, flags);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return retval;
}


/*-------------------------------------------------------------------------*/

/* endpoint set/clear halt */
static void ep_set_halt(struct langwell_ep *ep, int value)
{
        u32                     endptctrl = 0;
        int                     ep_num;
        struct langwell_udc     *dev = ep->dev;
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        ep_num = ep->ep_num;
        endptctrl = readl(&dev->op_regs->endptctrl[ep_num]);

        /* value: 1 - set halt, 0 - clear halt */
        if (value) {
                /* set the stall bit */
                if (is_in(ep))
                        endptctrl |= EPCTRL_TXS;
                else
                        endptctrl |= EPCTRL_RXS;
        } else {
                /* clear the stall bit and reset data toggle */
                if (is_in(ep)) {
                        endptctrl &= ~EPCTRL_TXS;
                        endptctrl |= EPCTRL_TXR;
                } else {
                        endptctrl &= ~EPCTRL_RXS;
                        endptctrl |= EPCTRL_RXR;
                }
        }

        writel(endptctrl, &dev->op_regs->endptctrl[ep_num]);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/* set the endpoint halt feature */
static int langwell_ep_set_halt(struct usb_ep *_ep, int value)
{
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;
        unsigned long           flags;
        int                     retval = 0;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !ep->desc)
                return -EINVAL;

        if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
                return -ESHUTDOWN;

        if (usb_endpoint_xfer_isoc(ep->desc))
                return  -EOPNOTSUPP;

        spin_lock_irqsave(&dev->lock, flags);

        /*
         * attempt to halt IN ep will fail if any transfer requests
         * are still queue
         */
        if (!list_empty(&ep->queue) && is_in(ep) && value) {

                /* IN endpoint FIFO holds bytes */
                dev_dbg(&dev->pdev->dev, "%s FIFO holds bytes\n", _ep->name);
                retval = -EAGAIN;
                goto done;
        }

        /* endpoint set/clear halt */
        if (ep->ep_num) {
                ep_set_halt(ep, value);
        } else { /* endpoint 0 */
                dev->ep0_state = WAIT_FOR_SETUP;
                dev->ep0_dir = USB_DIR_OUT;
        }
done:
        spin_unlock_irqrestore(&dev->lock, flags);
        dev_dbg(&dev->pdev->dev, "%s %s halt\n",
                        _ep->name, value ? "set" : "clear");
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return retval;
}


/* set the halt feature and ignores clear requests */
static int langwell_ep_set_wedge(struct usb_ep *_ep)
{
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !ep->desc)
                return -EINVAL;

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return usb_ep_set_halt(_ep);
}


/* flush contents of a fifo */
static void langwell_ep_fifo_flush(struct usb_ep *_ep)
{
        struct langwell_ep      *ep;
        struct langwell_udc     *dev;
        u32                     flush_bit;
        unsigned long           timeout;

        ep = container_of(_ep, struct langwell_ep, ep);
        dev = ep->dev;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (!_ep || !ep->desc) {
                dev_vdbg(&dev->pdev->dev, "ep or ep->desc is NULL\n");
                dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
                return;
        }

        dev_vdbg(&dev->pdev->dev, "%s-%s fifo flush\n",
                        _ep->name, DIR_STRING(ep));

        /* flush endpoint buffer */
        if (ep->ep_num == 0)
                flush_bit = (1 << 16) | 1;
        else if (is_in(ep))
                flush_bit = 1 << (ep->ep_num + 16);     /* TX */
        else
                flush_bit = 1 << ep->ep_num;            /* RX */

        /* wait until flush complete */
        timeout = jiffies + FLUSH_TIMEOUT;
        do {
                writel(flush_bit, &dev->op_regs->endptflush);
                while (readl(&dev->op_regs->endptflush)) {
                        if (time_after(jiffies, timeout)) {
                                dev_err(&dev->pdev->dev, "ep flush timeout\n");
                                goto done;
                        }
                        cpu_relax();
                }
        } while (readl(&dev->op_regs->endptstat) & flush_bit);
done:
        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/* endpoints operations structure */
static const struct usb_ep_ops langwell_ep_ops = {

        /* configure endpoint, making it usable */
        .enable         = langwell_ep_enable,

        /* endpoint is no longer usable */
        .disable        = langwell_ep_disable,

        /* allocate a request object to use with this endpoint */
        .alloc_request  = langwell_alloc_request,

        /* free a request object */
        .free_request   = langwell_free_request,

        /* queue (submits) an I/O requests to an endpoint */
        .queue          = langwell_ep_queue,

        /* dequeue (cancels, unlinks) an I/O request from an endpoint */
        .dequeue        = langwell_ep_dequeue,

        /* set the endpoint halt feature */
        .set_halt       = langwell_ep_set_halt,

        /* set the halt feature and ignores clear requests */
        .set_wedge      = langwell_ep_set_wedge,

        /* flush contents of a fifo */
        .fifo_flush     = langwell_ep_fifo_flush,
};


/*-------------------------------------------------------------------------*/

/* device controller usb_gadget_ops structure */

/* returns the current frame number */
static int langwell_get_frame(struct usb_gadget *_gadget)
{
        struct langwell_udc     *dev;
        u16                     retval;

        if (!_gadget)
                return -ENODEV;

        dev = container_of(_gadget, struct langwell_udc, gadget);
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        retval = readl(&dev->op_regs->frindex) & FRINDEX_MASK;

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return retval;
}


/* enter or exit PHY low power state */
static void langwell_phy_low_power(struct langwell_udc *dev, bool flag)
{
        u32             devlc;
        u8              devlc_byte2;
        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        devlc = readl(&dev->op_regs->devlc);
        dev_vdbg(&dev->pdev->dev, "devlc = 0x%08x\n", devlc);

        if (flag)
                devlc |= LPM_PHCD;
        else
                devlc &= ~LPM_PHCD;

        /* FIXME: workaround for Langwell A1/A2/A3 sighting */
        devlc_byte2 = (devlc >> 16) & 0xff;
        writeb(devlc_byte2, (u8 *)&dev->op_regs->devlc + 2);

        devlc = readl(&dev->op_regs->devlc);
        dev_vdbg(&dev->pdev->dev,
                        "%s PHY low power suspend, devlc = 0x%08x\n",
                        flag ? "enter" : "exit", devlc);
}


/* tries to wake up the host connected to this gadget */
static int langwell_wakeup(struct usb_gadget *_gadget)
{
        struct langwell_udc     *dev;
        u32                     portsc1;
        unsigned long           flags;

        if (!_gadget)
                return 0;

        dev = container_of(_gadget, struct langwell_udc, gadget);
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* remote wakeup feature not enabled by host */
        if (!dev->remote_wakeup) {
                dev_info(&dev->pdev->dev, "remote wakeup is disabled\n");
                return -ENOTSUPP;
        }

        spin_lock_irqsave(&dev->lock, flags);

        portsc1 = readl(&dev->op_regs->portsc1);
        if (!(portsc1 & PORTS_SUSP)) {
                spin_unlock_irqrestore(&dev->lock, flags);
                return 0;
        }

        /* LPM L1 to L0 or legacy remote wakeup */
        if (dev->lpm && dev->lpm_state == LPM_L1)
                dev_info(&dev->pdev->dev, "LPM L1 to L0 remote wakeup\n");
        else
                dev_info(&dev->pdev->dev, "device remote wakeup\n");

        /* exit PHY low power suspend */
        if (dev->pdev->device != 0x0829)
                langwell_phy_low_power(dev, 0);

        /* force port resume */
        portsc1 |= PORTS_FPR;
        writel(portsc1, &dev->op_regs->portsc1);

        spin_unlock_irqrestore(&dev->lock, flags);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}


/* notify controller that VBUS is powered or not */
static int langwell_vbus_session(struct usb_gadget *_gadget, int is_active)
{
        struct langwell_udc     *dev;
        unsigned long           flags;
        u32                     usbcmd;

        if (!_gadget)
                return -ENODEV;

        dev = container_of(_gadget, struct langwell_udc, gadget);
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        spin_lock_irqsave(&dev->lock, flags);
        dev_vdbg(&dev->pdev->dev, "VBUS status: %s\n",
                        is_active ? "on" : "off");

        dev->vbus_active = (is_active != 0);
        if (dev->driver && dev->softconnected && dev->vbus_active) {
                usbcmd = readl(&dev->op_regs->usbcmd);
                usbcmd |= CMD_RUNSTOP;
                writel(usbcmd, &dev->op_regs->usbcmd);
        } else {
                usbcmd = readl(&dev->op_regs->usbcmd);
                usbcmd &= ~CMD_RUNSTOP;
                writel(usbcmd, &dev->op_regs->usbcmd);
        }

        spin_unlock_irqrestore(&dev->lock, flags);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}


/* constrain controller's VBUS power usage */
static int langwell_vbus_draw(struct usb_gadget *_gadget, unsigned mA)
{
        struct langwell_udc     *dev;

        if (!_gadget)
                return -ENODEV;

        dev = container_of(_gadget, struct langwell_udc, gadget);
        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        if (dev->transceiver) {
                dev_vdbg(&dev->pdev->dev, "otg_set_power\n");
                dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
                return otg_set_power(dev->transceiver, mA);
        }

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return -ENOTSUPP;
}


/* D+ pullup, software-controlled connect/disconnect to USB host */
static int langwell_pullup(struct usb_gadget *_gadget, int is_on)
{
        struct langwell_udc     *dev;
        u32                     usbcmd;
        unsigned long           flags;

        if (!_gadget)
                return -ENODEV;

        dev = container_of(_gadget, struct langwell_udc, gadget);

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        spin_lock_irqsave(&dev->lock, flags);
        dev->softconnected = (is_on != 0);

        if (dev->driver && dev->softconnected && dev->vbus_active) {
                usbcmd = readl(&dev->op_regs->usbcmd);
                usbcmd |= CMD_RUNSTOP;
                writel(usbcmd, &dev->op_regs->usbcmd);
        } else {
                usbcmd = readl(&dev->op_regs->usbcmd);
                usbcmd &= ~CMD_RUNSTOP;
                writel(usbcmd, &dev->op_regs->usbcmd);
        }
        spin_unlock_irqrestore(&dev->lock, flags);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}

static int langwell_start(struct usb_gadget *g,
                struct usb_gadget_driver *driver);

static int langwell_stop(struct usb_gadget *g,
                struct usb_gadget_driver *driver);

/* device controller usb_gadget_ops structure */
static const struct usb_gadget_ops langwell_ops = {

        /* returns the current frame number */
        .get_frame      = langwell_get_frame,

        /* tries to wake up the host connected to this gadget */
        .wakeup         = langwell_wakeup,

        /* set the device selfpowered feature, always selfpowered */
        /* .set_selfpowered = langwell_set_selfpowered, */

        /* notify controller that VBUS is powered or not */
        .vbus_session   = langwell_vbus_session,

        /* constrain controller's VBUS power usage */
        .vbus_draw      = langwell_vbus_draw,

        /* D+ pullup, software-controlled connect/disconnect to USB host */
        .pullup         = langwell_pullup,

        .udc_start      = langwell_start,
        .udc_stop       = langwell_stop,
};


/*-------------------------------------------------------------------------*/

/* device controller operations */

/* reset device controller */
static int langwell_udc_reset(struct langwell_udc *dev)
{
        u32             usbcmd, usbmode, devlc, endpointlistaddr;
        u8              devlc_byte0, devlc_byte2;
        unsigned long   timeout;

        if (!dev)
                return -EINVAL;

        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* set controller to stop state */
        usbcmd = readl(&dev->op_regs->usbcmd);
        usbcmd &= ~CMD_RUNSTOP;
        writel(usbcmd, &dev->op_regs->usbcmd);

        /* reset device controller */
        usbcmd = readl(&dev->op_regs->usbcmd);
        usbcmd |= CMD_RST;
        writel(usbcmd, &dev->op_regs->usbcmd);

        /* wait for reset to complete */
        timeout = jiffies + RESET_TIMEOUT;
        while (readl(&dev->op_regs->usbcmd) & CMD_RST) {
                if (time_after(jiffies, timeout)) {
                        dev_err(&dev->pdev->dev, "device reset timeout\n");
                        return -ETIMEDOUT;
                }
                cpu_relax();
        }

        /* set controller to device mode */
        usbmode = readl(&dev->op_regs->usbmode);
        usbmode |= MODE_DEVICE;

        /* turn setup lockout off, require setup tripwire in usbcmd */
        usbmode |= MODE_SLOM;

        writel(usbmode, &dev->op_regs->usbmode);
        usbmode = readl(&dev->op_regs->usbmode);
        dev_vdbg(&dev->pdev->dev, "usbmode=0x%08x\n", usbmode);

        /* Write-Clear setup status */
        writel(0, &dev->op_regs->usbsts);

        /* if support USB LPM, ACK all LPM token */
        if (dev->lpm) {
                devlc = readl(&dev->op_regs->devlc);
                dev_vdbg(&dev->pdev->dev, "devlc = 0x%08x\n", devlc);
                /* FIXME: workaround for Langwell A1/A2/A3 sighting */
                devlc &= ~LPM_STL;      /* don't STALL LPM token */
                devlc &= ~LPM_NYT_ACK;  /* ACK LPM token */
                devlc_byte0 = devlc & 0xff;
                devlc_byte2 = (devlc >> 16) & 0xff;
                writeb(devlc_byte0, (u8 *)&dev->op_regs->devlc);
                writeb(devlc_byte2, (u8 *)&dev->op_regs->devlc + 2);
                devlc = readl(&dev->op_regs->devlc);
                dev_vdbg(&dev->pdev->dev,
                                "ACK LPM token, devlc = 0x%08x\n", devlc);
        }

        /* fill endpointlistaddr register */
        endpointlistaddr = dev->ep_dqh_dma;
        endpointlistaddr &= ENDPOINTLISTADDR_MASK;
        writel(endpointlistaddr, &dev->op_regs->endpointlistaddr);

        dev_vdbg(&dev->pdev->dev,
                "dQH base (vir: %p, phy: 0x%08x), endpointlistaddr=0x%08x\n",
                dev->ep_dqh, endpointlistaddr,
                readl(&dev->op_regs->endpointlistaddr));
        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}


/* reinitialize device controller endpoints */
static int eps_reinit(struct langwell_udc *dev)
{
        struct langwell_ep      *ep;
        char                    name[14];
        int                     i;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* initialize ep0 */
        ep = &dev->ep[0];
        ep->dev = dev;
        strncpy(ep->name, "ep0", sizeof(ep->name));
        ep->ep.name = ep->name;
        ep->ep.ops = &langwell_ep_ops;
        ep->stopped = 0;
        ep->ep.maxpacket = EP0_MAX_PKT_SIZE;
        ep->ep_num = 0;
        ep->desc = &langwell_ep0_desc;
        INIT_LIST_HEAD(&ep->queue);

        ep->ep_type = USB_ENDPOINT_XFER_CONTROL;

        /* initialize other endpoints */
        for (i = 2; i < dev->ep_max; i++) {
                ep = &dev->ep[i];
                if (i % 2)
                        snprintf(name, sizeof(name), "ep%din", i / 2);
                else
                        snprintf(name, sizeof(name), "ep%dout", i / 2);
                ep->dev = dev;
                strncpy(ep->name, name, sizeof(ep->name));
                ep->ep.name = ep->name;

                ep->ep.ops = &langwell_ep_ops;
                ep->stopped = 0;
                ep->ep.maxpacket = (unsigned short) ~0;
                ep->ep_num = i / 2;

                INIT_LIST_HEAD(&ep->queue);
                list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
        }

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
        return 0;
}


/* enable interrupt and set controller to run state */
static void langwell_udc_start(struct langwell_udc *dev)
{
        u32     usbintr, usbcmd;
        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* enable interrupts */
        usbintr = INTR_ULPIE    /* ULPI */
                | INTR_SLE      /* suspend */
                /* | INTR_SRE   SOF received */
                | INTR_URE      /* USB reset */
                | INTR_AAE      /* async advance */
                | INTR_SEE      /* system error */
                | INTR_FRE      /* frame list rollover */
                | INTR_PCE      /* port change detect */
                | INTR_UEE      /* USB error interrupt */
                | INTR_UE;      /* USB interrupt */
        writel(usbintr, &dev->op_regs->usbintr);

        /* clear stopped bit */
        dev->stopped = 0;

        /* set controller to run */
        usbcmd = readl(&dev->op_regs->usbcmd);
        usbcmd |= CMD_RUNSTOP;
        writel(usbcmd, &dev->op_regs->usbcmd);

        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/* disable interrupt and set controller to stop state */
static void langwell_udc_stop(struct langwell_udc *dev)
{
        u32     usbcmd;

        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* disable all interrupts */
        writel(0, &dev->op_regs->usbintr);

        /* set stopped bit */
        dev->stopped = 1;

        /* set controller to stop state */
        usbcmd = readl(&dev->op_regs->usbcmd);
        usbcmd &= ~CMD_RUNSTOP;
        writel(usbcmd, &dev->op_regs->usbcmd);

        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/* stop all USB activities */
static void stop_activity(struct langwell_udc *dev)
{
        struct langwell_ep      *ep;
        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        nuke(&dev->ep[0], -ESHUTDOWN);

        list_for_each_entry(ep, &dev->gadget.ep_list, ep.ep_list) {
                nuke(ep, -ESHUTDOWN);
        }

        /* report disconnect; the driver is already quiesced */
        if (dev->driver) {
                spin_unlock(&dev->lock);
                dev->driver->disconnect(&dev->gadget);
                spin_lock(&dev->lock);
        }

        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/*-------------------------------------------------------------------------*/

/* device "function" sysfs attribute file */
static ssize_t show_function(struct device *_dev,
                struct device_attribute *attr, char *buf)
{
        struct langwell_udc     *dev = dev_get_drvdata(_dev);

        if (!dev->driver || !dev->driver->function
                        || strlen(dev->driver->function) > PAGE_SIZE)
                return 0;

        return scnprintf(buf, PAGE_SIZE, "%s\n", dev->driver->function);
}
static DEVICE_ATTR(function, S_IRUGO, show_function, NULL);


static inline enum usb_device_speed lpm_device_speed(u32 reg)
{
        switch (LPM_PSPD(reg)) {
        case LPM_SPEED_HIGH:
                return USB_SPEED_HIGH;
        case LPM_SPEED_FULL:
                return USB_SPEED_FULL;
        case LPM_SPEED_LOW:
                return USB_SPEED_LOW;
        default:
                return USB_SPEED_UNKNOWN;
        }
}

/* device "langwell_udc" sysfs attribute file */
static ssize_t show_langwell_udc(struct device *_dev,
                struct device_attribute *attr, char *buf)
{
        struct langwell_udc     *dev = dev_get_drvdata(_dev);
        struct langwell_request *req;
        struct langwell_ep      *ep = NULL;
        char                    *next;
        unsigned                size;
        unsigned                t;
        unsigned                i;
        unsigned long           flags;
        u32                     tmp_reg;

        next = buf;
        size = PAGE_SIZE;
        spin_lock_irqsave(&dev->lock, flags);

        /* driver basic information */
        t = scnprintf(next, size,
                        DRIVER_DESC "\n"
                        "%s version: %s\n"
                        "Gadget driver: %s\n\n",
                        driver_name, DRIVER_VERSION,
                        dev->driver ? dev->driver->driver.name : "(none)");
        size -= t;
        next += t;

        /* device registers */
        tmp_reg = readl(&dev->op_regs->usbcmd);
        t = scnprintf(next, size,
                        "USBCMD reg:\n"
                        "SetupTW: %d\n"
                        "Run/Stop: %s\n\n",
                        (tmp_reg & CMD_SUTW) ? 1 : 0,
                        (tmp_reg & CMD_RUNSTOP) ? "Run" : "Stop");
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->usbsts);
        t = scnprintf(next, size,
                        "USB Status Reg:\n"
                        "Device Suspend: %d\n"
                        "Reset Received: %d\n"
                        "System Error: %s\n"
                        "USB Error Interrupt: %s\n\n",
                        (tmp_reg & STS_SLI) ? 1 : 0,
                        (tmp_reg & STS_URI) ? 1 : 0,
                        (tmp_reg & STS_SEI) ? "Error" : "No error",
                        (tmp_reg & STS_UEI) ? "Error detected" : "No error");
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->usbintr);
        t = scnprintf(next, size,
                        "USB Intrrupt Enable Reg:\n"
                        "Sleep Enable: %d\n"
                        "SOF Received Enable: %d\n"
                        "Reset Enable: %d\n"
                        "System Error Enable: %d\n"
                        "Port Change Dectected Enable: %d\n"
                        "USB Error Intr Enable: %d\n"
                        "USB Intr Enable: %d\n\n",
                        (tmp_reg & INTR_SLE) ? 1 : 0,
                        (tmp_reg & INTR_SRE) ? 1 : 0,
                        (tmp_reg & INTR_URE) ? 1 : 0,
                        (tmp_reg & INTR_SEE) ? 1 : 0,
                        (tmp_reg & INTR_PCE) ? 1 : 0,
                        (tmp_reg & INTR_UEE) ? 1 : 0,
                        (tmp_reg & INTR_UE) ? 1 : 0);
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->frindex);
        t = scnprintf(next, size,
                        "USB Frame Index Reg:\n"
                        "Frame Number is 0x%08x\n\n",
                        (tmp_reg & FRINDEX_MASK));
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->deviceaddr);
        t = scnprintf(next, size,
                        "USB Device Address Reg:\n"
                        "Device Addr is 0x%x\n\n",
                        USBADR(tmp_reg));
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->endpointlistaddr);
        t = scnprintf(next, size,
                        "USB Endpoint List Address Reg:\n"
                        "Endpoint List Pointer is 0x%x\n\n",
                        EPBASE(tmp_reg));
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->portsc1);
        t = scnprintf(next, size,
                "USB Port Status & Control Reg:\n"
                "Port Reset: %s\n"
                "Port Suspend Mode: %s\n"
                "Over-current Change: %s\n"
                "Port Enable/Disable Change: %s\n"
                "Port Enabled/Disabled: %s\n"
                "Current Connect Status: %s\n"
                "LPM Suspend Status: %s\n\n",
                (tmp_reg & PORTS_PR) ? "Reset" : "Not Reset",
                (tmp_reg & PORTS_SUSP) ? "Suspend " : "Not Suspend",
                (tmp_reg & PORTS_OCC) ? "Detected" : "No",
                (tmp_reg & PORTS_PEC) ? "Changed" : "Not Changed",
                (tmp_reg & PORTS_PE) ? "Enable" : "Not Correct",
                (tmp_reg & PORTS_CCS) ?  "Attached" : "Not Attached",
                (tmp_reg & PORTS_SLP) ? "LPM L1" : "LPM L0");
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->devlc);
        t = scnprintf(next, size,
                "Device LPM Control Reg:\n"
                "Parallel Transceiver : %d\n"
                "Serial Transceiver : %d\n"
                "Port Speed: %s\n"
                "Port Force Full Speed Connenct: %s\n"
                "PHY Low Power Suspend Clock: %s\n"
                "BmAttributes: %d\n\n",
                LPM_PTS(tmp_reg),
                (tmp_reg & LPM_STS) ? 1 : 0,
                usb_speed_string(lpm_device_speed(tmp_reg)),
                (tmp_reg & LPM_PFSC) ? "Force Full Speed" : "Not Force",
                (tmp_reg & LPM_PHCD) ? "Disabled" : "Enabled",
                LPM_BA(tmp_reg));
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->usbmode);
        t = scnprintf(next, size,
                        "USB Mode Reg:\n"
                        "Controller Mode is : %s\n\n", ({
                                char *s;
                                switch (MODE_CM(tmp_reg)) {
                                case MODE_IDLE:
                                        s = "Idle"; break;
                                case MODE_DEVICE:
                                        s = "Device Controller"; break;
                                case MODE_HOST:
                                        s = "Host Controller"; break;
                                default:
                                        s = "None"; break;
                                }
                                s;
                        }));
        size -= t;
        next += t;

        tmp_reg = readl(&dev->op_regs->endptsetupstat);
        t = scnprintf(next, size,
                        "Endpoint Setup Status Reg:\n"
                        "SETUP on ep 0x%04x\n\n",
                        tmp_reg & SETUPSTAT_MASK);
        size -= t;
        next += t;

        for (i = 0; i < dev->ep_max / 2; i++) {
                tmp_reg = readl(&dev->op_regs->endptctrl[i]);
                t = scnprintf(next, size, "EP Ctrl Reg [%d]: 0x%08x\n",
                                i, tmp_reg);
                size -= t;
                next += t;
        }
        tmp_reg = readl(&dev->op_regs->endptprime);
        t = scnprintf(next, size, "EP Prime Reg: 0x%08x\n\n", tmp_reg);
        size -= t;
        next += t;

        /* langwell_udc, langwell_ep, langwell_request structure information */
        ep = &dev->ep[0];
        t = scnprintf(next, size, "%s MaxPacketSize: 0x%x, ep_num: %d\n",
                        ep->ep.name, ep->ep.maxpacket, ep->ep_num);
        size -= t;
        next += t;

        if (list_empty(&ep->queue)) {
                t = scnprintf(next, size, "its req queue is empty\n\n");
                size -= t;
                next += t;
        } else {
                list_for_each_entry(req, &ep->queue, queue) {
                        t = scnprintf(next, size,
                                "req %p actual 0x%x length 0x%x  buf %p\n",
                                &req->req, req->req.actual,
                                req->req.length, req->req.buf);
                        size -= t;
                        next += t;
                }
        }
        /* other gadget->eplist ep */
        list_for_each_entry(ep, &dev->gadget.ep_list, ep.ep_list) {
                if (ep->desc) {
                        t = scnprintf(next, size,
                                        "\n%s MaxPacketSize: 0x%x, "
                                        "ep_num: %d\n",
                                        ep->ep.name, ep->ep.maxpacket,
                                        ep->ep_num);
                        size -= t;
                        next += t;

                        if (list_empty(&ep->queue)) {
                                t = scnprintf(next, size,
                                                "its req queue is empty\n\n");
                                size -= t;
                                next += t;
                        } else {
                                list_for_each_entry(req, &ep->queue, queue) {
                                        t = scnprintf(next, size,
                                                "req %p actual 0x%x length "
                                                "0x%x  buf %p\n",
                                                &req->req, req->req.actual,
                                                req->req.length, req->req.buf);
                                        size -= t;
                                        next += t;
                                }
                        }
                }
        }

        spin_unlock_irqrestore(&dev->lock, flags);
        return PAGE_SIZE - size;
}
static DEVICE_ATTR(langwell_udc, S_IRUGO, show_langwell_udc, NULL);


/* device "remote_wakeup" sysfs attribute file */
static ssize_t store_remote_wakeup(struct device *_dev,
                struct device_attribute *attr, const char *buf, size_t count)
{
        struct langwell_udc     *dev = dev_get_drvdata(_dev);
        unsigned long           flags;
        ssize_t                 rc = count;

        if (count > 2)
                return -EINVAL;

        if (count > 0 && buf[count-1] == '\n')
                ((char *) buf)[count-1] = 0;

        if (buf[0] != '1')
                return -EINVAL;

        /* force remote wakeup enabled in case gadget driver doesn't support */
        spin_lock_irqsave(&dev->lock, flags);
        dev->remote_wakeup = 1;
        dev->dev_status |= (1 << USB_DEVICE_REMOTE_WAKEUP);
        spin_unlock_irqrestore(&dev->lock, flags);

        langwell_wakeup(&dev->gadget);

        return rc;
}
static DEVICE_ATTR(remote_wakeup, S_IWUSR, NULL, store_remote_wakeup);


/*-------------------------------------------------------------------------*/

/*
 * when a driver is successfully registered, it will receive
 * control requests including set_configuration(), which enables
 * non-control requests.  then usb traffic follows until a
 * disconnect is reported.  then a host may connect again, or
 * the driver might get unbound.
 */

static int langwell_start(struct usb_gadget *g,
                struct usb_gadget_driver *driver)
{
        struct langwell_udc     *dev = gadget_to_langwell(g);
        unsigned long           flags;
        int                     retval;

        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        spin_lock_irqsave(&dev->lock, flags);

        /* hook up the driver ... */
        driver->driver.bus = NULL;
        dev->driver = driver;
        dev->gadget.dev.driver = &driver->driver;

        spin_unlock_irqrestore(&dev->lock, flags);

        retval = device_create_file(&dev->pdev->dev, &dev_attr_function);
        if (retval)
                goto err;

        dev->usb_state = USB_STATE_ATTACHED;
        dev->ep0_state = WAIT_FOR_SETUP;
        dev->ep0_dir = USB_DIR_OUT;

        /* enable interrupt and set controller to run state */
        if (dev->got_irq)
                langwell_udc_start(dev);

        dev_vdbg(&dev->pdev->dev,
                        "After langwell_udc_start(), print all registers:\n");
        print_all_registers(dev);

        dev_info(&dev->pdev->dev, "register driver: %s\n",
                        driver->driver.name);
        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);

        return 0;

err:
        dev->gadget.dev.driver = NULL;
        dev->driver = NULL;

        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);

        return retval;
}

/* unregister gadget driver */
static int langwell_stop(struct usb_gadget *g,
                struct usb_gadget_driver *driver)
{
        struct langwell_udc     *dev = gadget_to_langwell(g);
        unsigned long           flags;

        dev_dbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* exit PHY low power suspend */
        if (dev->pdev->device != 0x0829)
                langwell_phy_low_power(dev, 0);

        /* unbind OTG transceiver */
        if (dev->transceiver)
                (void)otg_set_peripheral(dev->transceiver, 0);

        /* disable interrupt and set controller to stop state */
        langwell_udc_stop(dev);

        dev->usb_state = USB_STATE_ATTACHED;
        dev->ep0_state = WAIT_FOR_SETUP;
        dev->ep0_dir = USB_DIR_OUT;

        spin_lock_irqsave(&dev->lock, flags);

        /* stop all usb activities */
        dev->gadget.speed = USB_SPEED_UNKNOWN;
        dev->gadget.dev.driver = NULL;
        dev->driver = NULL;
        stop_activity(dev);
        spin_unlock_irqrestore(&dev->lock, flags);

        device_remove_file(&dev->pdev->dev, &dev_attr_function);

        dev_info(&dev->pdev->dev, "unregistered driver '%s'\n",
                        driver->driver.name);
        dev_dbg(&dev->pdev->dev, "<--- %s()\n", __func__);

        return 0;
}

/*-------------------------------------------------------------------------*/

/*
 * setup tripwire is used as a semaphore to ensure that the setup data
 * payload is extracted from a dQH without being corrupted
 */
static void setup_tripwire(struct langwell_udc *dev)
{
        u32                     usbcmd,
                                endptsetupstat;
        unsigned long           timeout;
        struct langwell_dqh     *dqh;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* ep0 OUT dQH */
        dqh = &dev->ep_dqh[EP_DIR_OUT];

        /* Write-Clear endptsetupstat */
        endptsetupstat = readl(&dev->op_regs->endptsetupstat);
        writel(endptsetupstat, &dev->op_regs->endptsetupstat);

        /* wait until endptsetupstat is cleared */
        timeout = jiffies + SETUPSTAT_TIMEOUT;
        while (readl(&dev->op_regs->endptsetupstat)) {
                if (time_after(jiffies, timeout)) {
                        dev_err(&dev->pdev->dev, "setup_tripwire timeout\n");
                        break;
                }
                cpu_relax();
        }

        /* while a hazard exists when setup packet arrives */
        do {
                /* set setup tripwire bit */
                usbcmd = readl(&dev->op_regs->usbcmd);
                writel(usbcmd | CMD_SUTW, &dev->op_regs->usbcmd);

                /* copy the setup packet to local buffer */
                memcpy(&dev->local_setup_buff, &dqh->dqh_setup, 8);
        } while (!(readl(&dev->op_regs->usbcmd) & CMD_SUTW));

        /* Write-Clear setup tripwire bit */
        usbcmd = readl(&dev->op_regs->usbcmd);
        writel(usbcmd & ~CMD_SUTW, &dev->op_regs->usbcmd);

        dev_vdbg(&dev->pdev->dev, "<--- %s()\n", __func__);
}


/* protocol ep0 stall, will automatically be cleared on new transaction */
static void ep0_stall(struct langwell_udc *dev)
{
        u32     endptctrl;

        dev_vdbg(&dev->pdev->dev, "---> %s()\n", __func__);

        /* set TX and RX to stall */
        endptctrl = readl(&dev->op_regs->endptctrl[0]);
        endptctrl |= EPCTRL_TXS | EPCTRL_RXS;
        writel(endptctrl, &dev->op_regs->endptctrl[0]);

        /* update ep0 state */
        dev->ep0_state = WAIT_FOR_SETUP;