linux/drivers/dma/ipu/ipu_idmac.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2008
   3 * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
   4 *
   5 * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 */
  11
  12#include <linux/init.h>
  13#include <linux/platform_device.h>
  14#include <linux/err.h>
  15#include <linux/spinlock.h>
  16#include <linux/delay.h>
  17#include <linux/list.h>
  18#include <linux/clk.h>
  19#include <linux/vmalloc.h>
  20#include <linux/string.h>
  21#include <linux/interrupt.h>
  22#include <linux/io.h>
  23
  24#include <mach/ipu.h>
  25
  26#include "ipu_intern.h"
  27
  28#define FS_VF_IN_VALID  0x00000002
  29#define FS_ENC_IN_VALID 0x00000001
  30
  31static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
  32                               bool wait_for_stop);
  33
  34/*
  35 * There can be only one, we could allocate it dynamically, but then we'd have
  36 * to add an extra parameter to some functions, and use something as ugly as
  37 *      struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
  38 * in the ISR
  39 */
  40static struct ipu ipu_data;
  41
  42#define to_ipu(id) container_of(id, struct ipu, idmac)
  43
  44static u32 __idmac_read_icreg(struct ipu *ipu, unsigned long reg)
  45{
  46        return __raw_readl(ipu->reg_ic + reg);
  47}
  48
  49#define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
  50
  51static void __idmac_write_icreg(struct ipu *ipu, u32 value, unsigned long reg)
  52{
  53        __raw_writel(value, ipu->reg_ic + reg);
  54}
  55
  56#define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
  57
  58static u32 idmac_read_ipureg(struct ipu *ipu, unsigned long reg)
  59{
  60        return __raw_readl(ipu->reg_ipu + reg);
  61}
  62
  63static void idmac_write_ipureg(struct ipu *ipu, u32 value, unsigned long reg)
  64{
  65        __raw_writel(value, ipu->reg_ipu + reg);
  66}
  67
  68/*****************************************************************************
  69 * IPU / IC common functions
  70 */
  71static void dump_idmac_reg(struct ipu *ipu)
  72{
  73        dev_dbg(ipu->dev, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
  74                "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
  75                idmac_read_icreg(ipu, IDMAC_CONF),
  76                idmac_read_icreg(ipu, IC_CONF),
  77                idmac_read_icreg(ipu, IDMAC_CHA_EN),
  78                idmac_read_icreg(ipu, IDMAC_CHA_PRI),
  79                idmac_read_icreg(ipu, IDMAC_CHA_BUSY));
  80        dev_dbg(ipu->dev, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
  81                "DB_MODE 0x%x, TASKS_STAT 0x%x\n",
  82                idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY),
  83                idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY),
  84                idmac_read_ipureg(ipu, IPU_CHA_CUR_BUF),
  85                idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL),
  86                idmac_read_ipureg(ipu, IPU_TASKS_STAT));
  87}
  88
  89static uint32_t bytes_per_pixel(enum pixel_fmt fmt)
  90{
  91        switch (fmt) {
  92        case IPU_PIX_FMT_GENERIC:       /* generic data */
  93        case IPU_PIX_FMT_RGB332:
  94        case IPU_PIX_FMT_YUV420P:
  95        case IPU_PIX_FMT_YUV422P:
  96        default:
  97                return 1;
  98        case IPU_PIX_FMT_RGB565:
  99        case IPU_PIX_FMT_YUYV:
 100        case IPU_PIX_FMT_UYVY:
 101                return 2;
 102        case IPU_PIX_FMT_BGR24:
 103        case IPU_PIX_FMT_RGB24:
 104                return 3;
 105        case IPU_PIX_FMT_GENERIC_32:    /* generic data */
 106        case IPU_PIX_FMT_BGR32:
 107        case IPU_PIX_FMT_RGB32:
 108        case IPU_PIX_FMT_ABGR32:
 109                return 4;
 110        }
 111}
 112
 113/* Enable direct write to memory by the Camera Sensor Interface */
 114static void ipu_ic_enable_task(struct ipu *ipu, enum ipu_channel channel)
 115{
 116        uint32_t ic_conf, mask;
 117
 118        switch (channel) {
 119        case IDMAC_IC_0:
 120                mask = IC_CONF_PRPENC_EN;
 121                break;
 122        case IDMAC_IC_7:
 123                mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN;
 124                break;
 125        default:
 126                return;
 127        }
 128        ic_conf = idmac_read_icreg(ipu, IC_CONF) | mask;
 129        idmac_write_icreg(ipu, ic_conf, IC_CONF);
 130}
 131
 132/* Called under spin_lock_irqsave(&ipu_data.lock) */
 133static void ipu_ic_disable_task(struct ipu *ipu, enum ipu_channel channel)
 134{
 135        uint32_t ic_conf, mask;
 136
 137        switch (channel) {
 138        case IDMAC_IC_0:
 139                mask = IC_CONF_PRPENC_EN;
 140                break;
 141        case IDMAC_IC_7:
 142                mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN;
 143                break;
 144        default:
 145                return;
 146        }
 147        ic_conf = idmac_read_icreg(ipu, IC_CONF) & ~mask;
 148        idmac_write_icreg(ipu, ic_conf, IC_CONF);
 149}
 150
 151static uint32_t ipu_channel_status(struct ipu *ipu, enum ipu_channel channel)
 152{
 153        uint32_t stat = TASK_STAT_IDLE;
 154        uint32_t task_stat_reg = idmac_read_ipureg(ipu, IPU_TASKS_STAT);
 155
 156        switch (channel) {
 157        case IDMAC_IC_7:
 158                stat = (task_stat_reg & TSTAT_CSI2MEM_MASK) >>
 159                        TSTAT_CSI2MEM_OFFSET;
 160                break;
 161        case IDMAC_IC_0:
 162        case IDMAC_SDC_0:
 163        case IDMAC_SDC_1:
 164        default:
 165                break;
 166        }
 167        return stat;
 168}
 169
 170struct chan_param_mem_planar {
 171        /* Word 0 */
 172        u32     xv:10;
 173        u32     yv:10;
 174        u32     xb:12;
 175
 176        u32     yb:12;
 177        u32     res1:2;
 178        u32     nsb:1;
 179        u32     lnpb:6;
 180        u32     ubo_l:11;
 181
 182        u32     ubo_h:15;
 183        u32     vbo_l:17;
 184
 185        u32     vbo_h:9;
 186        u32     res2:3;
 187        u32     fw:12;
 188        u32     fh_l:8;
 189
 190        u32     fh_h:4;
 191        u32     res3:28;
 192
 193        /* Word 1 */
 194        u32     eba0;
 195
 196        u32     eba1;
 197
 198        u32     bpp:3;
 199        u32     sl:14;
 200        u32     pfs:3;
 201        u32     bam:3;
 202        u32     res4:2;
 203        u32     npb:6;
 204        u32     res5:1;
 205
 206        u32     sat:2;
 207        u32     res6:30;
 208} __attribute__ ((packed));
 209
 210struct chan_param_mem_interleaved {
 211        /* Word 0 */
 212        u32     xv:10;
 213        u32     yv:10;
 214        u32     xb:12;
 215
 216        u32     yb:12;
 217        u32     sce:1;
 218        u32     res1:1;
 219        u32     nsb:1;
 220        u32     lnpb:6;
 221        u32     sx:10;
 222        u32     sy_l:1;
 223
 224        u32     sy_h:9;
 225        u32     ns:10;
 226        u32     sm:10;
 227        u32     sdx_l:3;
 228
 229        u32     sdx_h:2;
 230        u32     sdy:5;
 231        u32     sdrx:1;
 232        u32     sdry:1;
 233        u32     sdr1:1;
 234        u32     res2:2;
 235        u32     fw:12;
 236        u32     fh_l:8;
 237
 238        u32     fh_h:4;
 239        u32     res3:28;
 240
 241        /* Word 1 */
 242        u32     eba0;
 243
 244        u32     eba1;
 245
 246        u32     bpp:3;
 247        u32     sl:14;
 248        u32     pfs:3;
 249        u32     bam:3;
 250        u32     res4:2;
 251        u32     npb:6;
 252        u32     res5:1;
 253
 254        u32     sat:2;
 255        u32     scc:1;
 256        u32     ofs0:5;
 257        u32     ofs1:5;
 258        u32     ofs2:5;
 259        u32     ofs3:5;
 260        u32     wid0:3;
 261        u32     wid1:3;
 262        u32     wid2:3;
 263
 264        u32     wid3:3;
 265        u32     dec_sel:1;
 266        u32     res6:28;
 267} __attribute__ ((packed));
 268
 269union chan_param_mem {
 270        struct chan_param_mem_planar            pp;
 271        struct chan_param_mem_interleaved       ip;
 272};
 273
 274static void ipu_ch_param_set_plane_offset(union chan_param_mem *params,
 275                                          u32 u_offset, u32 v_offset)
 276{
 277        params->pp.ubo_l = u_offset & 0x7ff;
 278        params->pp.ubo_h = u_offset >> 11;
 279        params->pp.vbo_l = v_offset & 0x1ffff;
 280        params->pp.vbo_h = v_offset >> 17;
 281}
 282
 283static void ipu_ch_param_set_size(union chan_param_mem *params,
 284                                  uint32_t pixel_fmt, uint16_t width,
 285                                  uint16_t height, uint16_t stride)
 286{
 287        u32 u_offset;
 288        u32 v_offset;
 289
 290        params->pp.fw           = width - 1;
 291        params->pp.fh_l         = height - 1;
 292        params->pp.fh_h         = (height - 1) >> 8;
 293        params->pp.sl           = stride - 1;
 294
 295        switch (pixel_fmt) {
 296        case IPU_PIX_FMT_GENERIC:
 297                /*Represents 8-bit Generic data */
 298                params->pp.bpp  = 3;
 299                params->pp.pfs  = 7;
 300                params->pp.npb  = 31;
 301                params->pp.sat  = 2;            /* SAT = use 32-bit access */
 302                break;
 303        case IPU_PIX_FMT_GENERIC_32:
 304                /*Represents 32-bit Generic data */
 305                params->pp.bpp  = 0;
 306                params->pp.pfs  = 7;
 307                params->pp.npb  = 7;
 308                params->pp.sat  = 2;            /* SAT = use 32-bit access */
 309                break;
 310        case IPU_PIX_FMT_RGB565:
 311                params->ip.bpp  = 2;
 312                params->ip.pfs  = 4;
 313                params->ip.npb  = 7;
 314                params->ip.sat  = 2;            /* SAT = 32-bit access */
 315                params->ip.ofs0 = 0;            /* Red bit offset */
 316                params->ip.ofs1 = 5;            /* Green bit offset */
 317                params->ip.ofs2 = 11;           /* Blue bit offset */
 318                params->ip.ofs3 = 16;           /* Alpha bit offset */
 319                params->ip.wid0 = 4;            /* Red bit width - 1 */
 320                params->ip.wid1 = 5;            /* Green bit width - 1 */
 321                params->ip.wid2 = 4;            /* Blue bit width - 1 */
 322                break;
 323        case IPU_PIX_FMT_BGR24:
 324                params->ip.bpp  = 1;            /* 24 BPP & RGB PFS */
 325                params->ip.pfs  = 4;
 326                params->ip.npb  = 7;
 327                params->ip.sat  = 2;            /* SAT = 32-bit access */
 328                params->ip.ofs0 = 0;            /* Red bit offset */
 329                params->ip.ofs1 = 8;            /* Green bit offset */
 330                params->ip.ofs2 = 16;           /* Blue bit offset */
 331                params->ip.ofs3 = 24;           /* Alpha bit offset */
 332                params->ip.wid0 = 7;            /* Red bit width - 1 */
 333                params->ip.wid1 = 7;            /* Green bit width - 1 */
 334                params->ip.wid2 = 7;            /* Blue bit width - 1 */
 335                break;
 336        case IPU_PIX_FMT_RGB24:
 337                params->ip.bpp  = 1;            /* 24 BPP & RGB PFS */
 338                params->ip.pfs  = 4;
 339                params->ip.npb  = 7;
 340                params->ip.sat  = 2;            /* SAT = 32-bit access */
 341                params->ip.ofs0 = 16;           /* Red bit offset */
 342                params->ip.ofs1 = 8;            /* Green bit offset */
 343                params->ip.ofs2 = 0;            /* Blue bit offset */
 344                params->ip.ofs3 = 24;           /* Alpha bit offset */
 345                params->ip.wid0 = 7;            /* Red bit width - 1 */
 346                params->ip.wid1 = 7;            /* Green bit width - 1 */
 347                params->ip.wid2 = 7;            /* Blue bit width - 1 */
 348                break;
 349        case IPU_PIX_FMT_BGRA32:
 350        case IPU_PIX_FMT_BGR32:
 351        case IPU_PIX_FMT_ABGR32:
 352                params->ip.bpp  = 0;
 353                params->ip.pfs  = 4;
 354                params->ip.npb  = 7;
 355                params->ip.sat  = 2;            /* SAT = 32-bit access */
 356                params->ip.ofs0 = 8;            /* Red bit offset */
 357                params->ip.ofs1 = 16;           /* Green bit offset */
 358                params->ip.ofs2 = 24;           /* Blue bit offset */
 359                params->ip.ofs3 = 0;            /* Alpha bit offset */
 360                params->ip.wid0 = 7;            /* Red bit width - 1 */
 361                params->ip.wid1 = 7;            /* Green bit width - 1 */
 362                params->ip.wid2 = 7;            /* Blue bit width - 1 */
 363                params->ip.wid3 = 7;            /* Alpha bit width - 1 */
 364                break;
 365        case IPU_PIX_FMT_RGBA32:
 366        case IPU_PIX_FMT_RGB32:
 367                params->ip.bpp  = 0;
 368                params->ip.pfs  = 4;
 369                params->ip.npb  = 7;
 370                params->ip.sat  = 2;            /* SAT = 32-bit access */
 371                params->ip.ofs0 = 24;           /* Red bit offset */
 372                params->ip.ofs1 = 16;           /* Green bit offset */
 373                params->ip.ofs2 = 8;            /* Blue bit offset */
 374                params->ip.ofs3 = 0;            /* Alpha bit offset */
 375                params->ip.wid0 = 7;            /* Red bit width - 1 */
 376                params->ip.wid1 = 7;            /* Green bit width - 1 */
 377                params->ip.wid2 = 7;            /* Blue bit width - 1 */
 378                params->ip.wid3 = 7;            /* Alpha bit width - 1 */
 379                break;
 380        case IPU_PIX_FMT_UYVY:
 381                params->ip.bpp  = 2;
 382                params->ip.pfs  = 6;
 383                params->ip.npb  = 7;
 384                params->ip.sat  = 2;            /* SAT = 32-bit access */
 385                break;
 386        case IPU_PIX_FMT_YUV420P2:
 387        case IPU_PIX_FMT_YUV420P:
 388                params->ip.bpp  = 3;
 389                params->ip.pfs  = 3;
 390                params->ip.npb  = 7;
 391                params->ip.sat  = 2;            /* SAT = 32-bit access */
 392                u_offset = stride * height;
 393                v_offset = u_offset + u_offset / 4;
 394                ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
 395                break;
 396        case IPU_PIX_FMT_YVU422P:
 397                params->ip.bpp  = 3;
 398                params->ip.pfs  = 2;
 399                params->ip.npb  = 7;
 400                params->ip.sat  = 2;            /* SAT = 32-bit access */
 401                v_offset = stride * height;
 402                u_offset = v_offset + v_offset / 2;
 403                ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
 404                break;
 405        case IPU_PIX_FMT_YUV422P:
 406                params->ip.bpp  = 3;
 407                params->ip.pfs  = 2;
 408                params->ip.npb  = 7;
 409                params->ip.sat  = 2;            /* SAT = 32-bit access */
 410                u_offset = stride * height;
 411                v_offset = u_offset + u_offset / 2;
 412                ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
 413                break;
 414        default:
 415                dev_err(ipu_data.dev,
 416                        "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt);
 417                break;
 418        }
 419
 420        params->pp.nsb = 1;
 421}
 422
 423static void ipu_ch_param_set_burst_size(union chan_param_mem *params,
 424                                        uint16_t burst_pixels)
 425{
 426        params->pp.npb = burst_pixels - 1;
 427}
 428
 429static void ipu_ch_param_set_buffer(union chan_param_mem *params,
 430                                    dma_addr_t buf0, dma_addr_t buf1)
 431{
 432        params->pp.eba0 = buf0;
 433        params->pp.eba1 = buf1;
 434}
 435
 436static void ipu_ch_param_set_rotation(union chan_param_mem *params,
 437                                      enum ipu_rotate_mode rotate)
 438{
 439        params->pp.bam = rotate;
 440}
 441
 442static void ipu_write_param_mem(uint32_t addr, uint32_t *data,
 443                                uint32_t num_words)
 444{
 445        for (; num_words > 0; num_words--) {
 446                dev_dbg(ipu_data.dev,
 447                        "write param mem - addr = 0x%08X, data = 0x%08X\n",
 448                        addr, *data);
 449                idmac_write_ipureg(&ipu_data, addr, IPU_IMA_ADDR);
 450                idmac_write_ipureg(&ipu_data, *data++, IPU_IMA_DATA);
 451                addr++;
 452                if ((addr & 0x7) == 5) {
 453                        addr &= ~0x7;   /* set to word 0 */
 454                        addr += 8;      /* increment to next row */
 455                }
 456        }
 457}
 458
 459static int calc_resize_coeffs(uint32_t in_size, uint32_t out_size,
 460                              uint32_t *resize_coeff,
 461                              uint32_t *downsize_coeff)
 462{
 463        uint32_t temp_size;
 464        uint32_t temp_downsize;
 465
 466        *resize_coeff   = 1 << 13;
 467        *downsize_coeff = 1 << 13;
 468
 469        /* Cannot downsize more than 8:1 */
 470        if (out_size << 3 < in_size)
 471                return -EINVAL;
 472
 473        /* compute downsizing coefficient */
 474        temp_downsize = 0;
 475        temp_size = in_size;
 476        while (temp_size >= out_size * 2 && temp_downsize < 2) {
 477                temp_size >>= 1;
 478                temp_downsize++;
 479        }
 480        *downsize_coeff = temp_downsize;
 481
 482        /*
 483         * compute resizing coefficient using the following formula:
 484         * resize_coeff = M*(SI -1)/(SO - 1)
 485         * where M = 2^13, SI - input size, SO - output size
 486         */
 487        *resize_coeff = (8192L * (temp_size - 1)) / (out_size - 1);
 488        if (*resize_coeff >= 16384L) {
 489                dev_err(ipu_data.dev, "Warning! Overflow on resize coeff.\n");
 490                *resize_coeff = 0x3FFF;
 491        }
 492
 493        dev_dbg(ipu_data.dev, "resizing from %u -> %u pixels, "
 494                "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size, out_size,
 495                *downsize_coeff, *resize_coeff >= 8192L ? 1 : 0,
 496                ((*resize_coeff & 0x1FFF) * 10000L) / 8192L, *resize_coeff);
 497
 498        return 0;
 499}
 500
 501static enum ipu_color_space format_to_colorspace(enum pixel_fmt fmt)
 502{
 503        switch (fmt) {
 504        case IPU_PIX_FMT_RGB565:
 505        case IPU_PIX_FMT_BGR24:
 506        case IPU_PIX_FMT_RGB24:
 507        case IPU_PIX_FMT_BGR32:
 508        case IPU_PIX_FMT_RGB32:
 509                return IPU_COLORSPACE_RGB;
 510        default:
 511                return IPU_COLORSPACE_YCBCR;
 512        }
 513}
 514
 515static int ipu_ic_init_prpenc(struct ipu *ipu,
 516                              union ipu_channel_param *params, bool src_is_csi)
 517{
 518        uint32_t reg, ic_conf;
 519        uint32_t downsize_coeff, resize_coeff;
 520        enum ipu_color_space in_fmt, out_fmt;
 521
 522        /* Setup vertical resizing */
 523        calc_resize_coeffs(params->video.in_height,
 524                            params->video.out_height,
 525                            &resize_coeff, &downsize_coeff);
 526        reg = (downsize_coeff << 30) | (resize_coeff << 16);
 527
 528        /* Setup horizontal resizing */
 529        calc_resize_coeffs(params->video.in_width,
 530                            params->video.out_width,
 531                            &resize_coeff, &downsize_coeff);
 532        reg |= (downsize_coeff << 14) | resize_coeff;
 533
 534        /* Setup color space conversion */
 535        in_fmt = format_to_colorspace(params->video.in_pixel_fmt);
 536        out_fmt = format_to_colorspace(params->video.out_pixel_fmt);
 537
 538        /*
 539         * Colourspace conversion unsupported yet - see _init_csc() in
 540         * Freescale sources
 541         */
 542        if (in_fmt != out_fmt) {
 543                dev_err(ipu->dev, "Colourspace conversion unsupported!\n");
 544                return -EOPNOTSUPP;
 545        }
 546
 547        idmac_write_icreg(ipu, reg, IC_PRP_ENC_RSC);
 548
 549        ic_conf = idmac_read_icreg(ipu, IC_CONF);
 550
 551        if (src_is_csi)
 552                ic_conf &= ~IC_CONF_RWS_EN;
 553        else
 554                ic_conf |= IC_CONF_RWS_EN;
 555
 556        idmac_write_icreg(ipu, ic_conf, IC_CONF);
 557
 558        return 0;
 559}
 560
 561static uint32_t dma_param_addr(uint32_t dma_ch)
 562{
 563        /* Channel Parameter Memory */
 564        return 0x10000 | (dma_ch << 4);
 565}
 566
 567static void ipu_channel_set_priority(struct ipu *ipu, enum ipu_channel channel,
 568                                     bool prio)
 569{
 570        u32 reg = idmac_read_icreg(ipu, IDMAC_CHA_PRI);
 571
 572        if (prio)
 573                reg |= 1UL << channel;
 574        else
 575                reg &= ~(1UL << channel);
 576
 577        idmac_write_icreg(ipu, reg, IDMAC_CHA_PRI);
 578
 579        dump_idmac_reg(ipu);
 580}
 581
 582static uint32_t ipu_channel_conf_mask(enum ipu_channel channel)
 583{
 584        uint32_t mask;
 585
 586        switch (channel) {
 587        case IDMAC_IC_0:
 588        case IDMAC_IC_7:
 589                mask = IPU_CONF_CSI_EN | IPU_CONF_IC_EN;
 590                break;
 591        case IDMAC_SDC_0:
 592        case IDMAC_SDC_1:
 593                mask = IPU_CONF_SDC_EN | IPU_CONF_DI_EN;
 594                break;
 595        default:
 596                mask = 0;
 597                break;
 598        }
 599
 600        return mask;
 601}
 602
 603/**
 604 * ipu_enable_channel() - enable an IPU channel.
 605 * @idmac:      IPU DMAC context.
 606 * @ichan:      IDMAC channel.
 607 * @return:     0 on success or negative error code on failure.
 608 */
 609static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan)
 610{
 611        struct ipu *ipu = to_ipu(idmac);
 612        enum ipu_channel channel = ichan->dma_chan.chan_id;
 613        uint32_t reg;
 614        unsigned long flags;
 615
 616        spin_lock_irqsave(&ipu->lock, flags);
 617
 618        /* Reset to buffer 0 */
 619        idmac_write_ipureg(ipu, 1UL << channel, IPU_CHA_CUR_BUF);
 620        ichan->active_buffer = 0;
 621        ichan->status = IPU_CHANNEL_ENABLED;
 622
 623        switch (channel) {
 624        case IDMAC_SDC_0:
 625        case IDMAC_SDC_1:
 626        case IDMAC_IC_7:
 627                ipu_channel_set_priority(ipu, channel, true);
 628        default:
 629                break;
 630        }
 631
 632        reg = idmac_read_icreg(ipu, IDMAC_CHA_EN);
 633
 634        idmac_write_icreg(ipu, reg | (1UL << channel), IDMAC_CHA_EN);
 635
 636        ipu_ic_enable_task(ipu, channel);
 637
 638        spin_unlock_irqrestore(&ipu->lock, flags);
 639        return 0;
 640}
 641
 642/**
 643 * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
 644 * @ichan:      IDMAC channel.
 645 * @pixel_fmt:  pixel format of buffer. Pixel format is a FOURCC ASCII code.
 646 * @width:      width of buffer in pixels.
 647 * @height:     height of buffer in pixels.
 648 * @stride:     stride length of buffer in pixels.
 649 * @rot_mode:   rotation mode of buffer. A rotation setting other than
 650 *              IPU_ROTATE_VERT_FLIP should only be used for input buffers of
 651 *              rotation channels.
 652 * @phyaddr_0:  buffer 0 physical address.
 653 * @phyaddr_1:  buffer 1 physical address. Setting this to a value other than
 654 *              NULL enables double buffering mode.
 655 * @return:     0 on success or negative error code on failure.
 656 */
 657static int ipu_init_channel_buffer(struct idmac_channel *ichan,
 658                                   enum pixel_fmt pixel_fmt,
 659                                   uint16_t width, uint16_t height,
 660                                   uint32_t stride,
 661                                   enum ipu_rotate_mode rot_mode,
 662                                   dma_addr_t phyaddr_0, dma_addr_t phyaddr_1)
 663{
 664        enum ipu_channel channel = ichan->dma_chan.chan_id;
 665        struct idmac *idmac = to_idmac(ichan->dma_chan.device);
 666        struct ipu *ipu = to_ipu(idmac);
 667        union chan_param_mem params = {};
 668        unsigned long flags;
 669        uint32_t reg;
 670        uint32_t stride_bytes;
 671
 672        stride_bytes = stride * bytes_per_pixel(pixel_fmt);
 673
 674        if (stride_bytes % 4) {
 675                dev_err(ipu->dev,
 676                        "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
 677                        stride, stride_bytes);
 678                return -EINVAL;
 679        }
 680
 681        /* IC channel's stride must be a multiple of 8 pixels */
 682        if ((channel <= IDMAC_IC_13) && (stride % 8)) {
 683                dev_err(ipu->dev, "Stride must be 8 pixel multiple\n");
 684                return -EINVAL;
 685        }
 686
 687        /* Build parameter memory data for DMA channel */
 688        ipu_ch_param_set_size(&params, pixel_fmt, width, height, stride_bytes);
 689        ipu_ch_param_set_buffer(&params, phyaddr_0, phyaddr_1);
 690        ipu_ch_param_set_rotation(&params, rot_mode);
 691        /* Some channels (rotation) have restriction on burst length */
 692        switch (channel) {
 693        case IDMAC_IC_7:        /* Hangs with burst 8, 16, other values
 694                                   invalid - Table 44-30 */
 695/*
 696                ipu_ch_param_set_burst_size(&params, 8);
 697 */
 698                break;
 699        case IDMAC_SDC_0:
 700        case IDMAC_SDC_1:
 701                /* In original code only IPU_PIX_FMT_RGB565 was setting burst */
 702                ipu_ch_param_set_burst_size(&params, 16);
 703                break;
 704        case IDMAC_IC_0:
 705        default:
 706                break;
 707        }
 708
 709        spin_lock_irqsave(&ipu->lock, flags);
 710
 711        ipu_write_param_mem(dma_param_addr(channel), (uint32_t *)&params, 10);
 712
 713        reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL);
 714
 715        if (phyaddr_1)
 716                reg |= 1UL << channel;
 717        else
 718                reg &= ~(1UL << channel);
 719
 720        idmac_write_ipureg(ipu, reg, IPU_CHA_DB_MODE_SEL);
 721
 722        ichan->status = IPU_CHANNEL_READY;
 723
 724        spin_unlock_irqrestore(&ipu->lock, flags);
 725
 726        return 0;
 727}
 728
 729/**
 730 * ipu_select_buffer() - mark a channel's buffer as ready.
 731 * @channel:    channel ID.
 732 * @buffer_n:   buffer number to mark ready.
 733 */
 734static void ipu_select_buffer(enum ipu_channel channel, int buffer_n)
 735{
 736        /* No locking - this is a write-one-to-set register, cleared by IPU */
 737        if (buffer_n == 0)
 738                /* Mark buffer 0 as ready. */
 739                idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF0_RDY);
 740        else
 741                /* Mark buffer 1 as ready. */
 742                idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF1_RDY);
 743}
 744
 745/**
 746 * ipu_update_channel_buffer() - update physical address of a channel buffer.
 747 * @ichan:      IDMAC channel.
 748 * @buffer_n:   buffer number to update.
 749 *              0 or 1 are the only valid values.
 750 * @phyaddr:    buffer physical address.
 751 */
 752/* Called under spin_lock(_irqsave)(&ichan->lock) */
 753static void ipu_update_channel_buffer(struct idmac_channel *ichan,
 754                                      int buffer_n, dma_addr_t phyaddr)
 755{
 756        enum ipu_channel channel = ichan->dma_chan.chan_id;
 757        uint32_t reg;
 758        unsigned long flags;
 759
 760        spin_lock_irqsave(&ipu_data.lock, flags);
 761
 762        if (buffer_n == 0) {
 763                reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
 764                if (reg & (1UL << channel)) {
 765                        ipu_ic_disable_task(&ipu_data, channel);
 766                        ichan->status = IPU_CHANNEL_READY;
 767                }
 768
 769                /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
 770                idmac_write_ipureg(&ipu_data, dma_param_addr(channel) +
 771                                   0x0008UL, IPU_IMA_ADDR);
 772                idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA);
 773        } else {
 774                reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
 775                if (reg & (1UL << channel)) {
 776                        ipu_ic_disable_task(&ipu_data, channel);
 777                        ichan->status = IPU_CHANNEL_READY;
 778                }
 779
 780                /* Check if double-buffering is already enabled */
 781                reg = idmac_read_ipureg(&ipu_data, IPU_CHA_DB_MODE_SEL);
 782
 783                if (!(reg & (1UL << channel)))
 784                        idmac_write_ipureg(&ipu_data, reg | (1UL << channel),
 785                                           IPU_CHA_DB_MODE_SEL);
 786
 787                /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
 788                idmac_write_ipureg(&ipu_data, dma_param_addr(channel) +
 789                                   0x0009UL, IPU_IMA_ADDR);
 790                idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA);
 791        }
 792
 793        spin_unlock_irqrestore(&ipu_data.lock, flags);
 794}
 795
 796/* Called under spin_lock_irqsave(&ichan->lock) */
 797static int ipu_submit_buffer(struct idmac_channel *ichan,
 798        struct idmac_tx_desc *desc, struct scatterlist *sg, int buf_idx)
 799{
 800        unsigned int chan_id = ichan->dma_chan.chan_id;
 801        struct device *dev = &ichan->dma_chan.dev->device;
 802
 803        if (async_tx_test_ack(&desc->txd))
 804                return -EINTR;
 805
 806        /*
 807         * On first invocation this shouldn't be necessary, the call to
 808         * ipu_init_channel_buffer() above will set addresses for us, so we
 809         * could make it conditional on status >= IPU_CHANNEL_ENABLED, but
 810         * doing it again shouldn't hurt either.
 811         */
 812        ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg));
 813
 814        ipu_select_buffer(chan_id, buf_idx);
 815        dev_dbg(dev, "Updated sg %p on channel 0x%x buffer %d\n",
 816                sg, chan_id, buf_idx);
 817
 818        return 0;
 819}
 820
 821/* Called under spin_lock_irqsave(&ichan->lock) */
 822static int ipu_submit_channel_buffers(struct idmac_channel *ichan,
 823                                      struct idmac_tx_desc *desc)
 824{
 825        struct scatterlist *sg;
 826        int i, ret = 0;
 827
 828        for (i = 0, sg = desc->sg; i < 2 && sg; i++) {
 829                if (!ichan->sg[i]) {
 830                        ichan->sg[i] = sg;
 831
 832                        ret = ipu_submit_buffer(ichan, desc, sg, i);
 833                        if (ret < 0)
 834                                return ret;
 835
 836                        sg = sg_next(sg);
 837                }
 838        }
 839
 840        return ret;
 841}
 842
 843static dma_cookie_t idmac_tx_submit(struct dma_async_tx_descriptor *tx)
 844{
 845        struct idmac_tx_desc *desc = to_tx_desc(tx);
 846        struct idmac_channel *ichan = to_idmac_chan(tx->chan);
 847        struct idmac *idmac = to_idmac(tx->chan->device);
 848        struct ipu *ipu = to_ipu(idmac);
 849        struct device *dev = &ichan->dma_chan.dev->device;
 850        dma_cookie_t cookie;
 851        unsigned long flags;
 852        int ret;
 853
 854        /* Sanity check */
 855        if (!list_empty(&desc->list)) {
 856                /* The descriptor doesn't belong to client */
 857                dev_err(dev, "Descriptor %p not prepared!\n", tx);
 858                return -EBUSY;
 859        }
 860
 861        mutex_lock(&ichan->chan_mutex);
 862
 863        async_tx_clear_ack(tx);
 864
 865        if (ichan->status < IPU_CHANNEL_READY) {
 866                struct idmac_video_param *video = &ichan->params.video;
 867                /*
 868                 * Initial buffer assignment - the first two sg-entries from
 869                 * the descriptor will end up in the IDMAC buffers
 870                 */
 871                dma_addr_t dma_1 = sg_is_last(desc->sg) ? 0 :
 872                        sg_dma_address(&desc->sg[1]);
 873
 874                WARN_ON(ichan->sg[0] || ichan->sg[1]);
 875
 876                cookie = ipu_init_channel_buffer(ichan,
 877                                                 video->out_pixel_fmt,
 878                                                 video->out_width,
 879                                                 video->out_height,
 880                                                 video->out_stride,
 881                                                 IPU_ROTATE_NONE,
 882                                                 sg_dma_address(&desc->sg[0]),
 883                                                 dma_1);
 884                if (cookie < 0)
 885                        goto out;
 886        }
 887
 888        dev_dbg(dev, "Submitting sg %p\n", &desc->sg[0]);
 889
 890        cookie = ichan->dma_chan.cookie;
 891
 892        if (++cookie < 0)
 893                cookie = 1;
 894
 895        /* from dmaengine.h: "last cookie value returned to client" */
 896        ichan->dma_chan.cookie = cookie;
 897        tx->cookie = cookie;
 898
 899        /* ipu->lock can be taken under ichan->lock, but not v.v. */
 900        spin_lock_irqsave(&ichan->lock, flags);
 901
 902        list_add_tail(&desc->list, &ichan->queue);
 903        /* submit_buffers() atomically verifies and fills empty sg slots */
 904        ret = ipu_submit_channel_buffers(ichan, desc);
 905
 906        spin_unlock_irqrestore(&ichan->lock, flags);
 907
 908        if (ret < 0) {
 909                cookie = ret;
 910                goto dequeue;
 911        }
 912
 913        if (ichan->status < IPU_CHANNEL_ENABLED) {
 914                ret = ipu_enable_channel(idmac, ichan);
 915                if (ret < 0) {
 916                        cookie = ret;
 917                        goto dequeue;
 918                }
 919        }
 920
 921        dump_idmac_reg(ipu);
 922
 923dequeue:
 924        if (cookie < 0) {
 925                spin_lock_irqsave(&ichan->lock, flags);
 926                list_del_init(&desc->list);
 927                spin_unlock_irqrestore(&ichan->lock, flags);
 928                tx->cookie = cookie;
 929                ichan->dma_chan.cookie = cookie;
 930        }
 931
 932out:
 933        mutex_unlock(&ichan->chan_mutex);
 934
 935        return cookie;
 936}
 937
 938/* Called with ichan->chan_mutex held */
 939static int idmac_desc_alloc(struct idmac_channel *ichan, int n)
 940{
 941        struct idmac_tx_desc *desc = vmalloc(n * sizeof(struct idmac_tx_desc));
 942        struct idmac *idmac = to_idmac(ichan->dma_chan.device);
 943
 944        if (!desc)
 945                return -ENOMEM;
 946
 947        /* No interrupts, just disable the tasklet for a moment */
 948        tasklet_disable(&to_ipu(idmac)->tasklet);
 949
 950        ichan->n_tx_desc = n;
 951        ichan->desc = desc;
 952        INIT_LIST_HEAD(&ichan->queue);
 953        INIT_LIST_HEAD(&ichan->free_list);
 954
 955        while (n--) {
 956                struct dma_async_tx_descriptor *txd = &desc->txd;
 957
 958                memset(txd, 0, sizeof(*txd));
 959                dma_async_tx_descriptor_init(txd, &ichan->dma_chan);
 960                txd->tx_submit          = idmac_tx_submit;
 961
 962                list_add(&desc->list, &ichan->free_list);
 963
 964                desc++;
 965        }
 966
 967        tasklet_enable(&to_ipu(idmac)->tasklet);
 968
 969        return 0;
 970}
 971
 972/**
 973 * ipu_init_channel() - initialize an IPU channel.
 974 * @idmac:      IPU DMAC context.
 975 * @ichan:      pointer to the channel object.
 976 * @return      0 on success or negative error code on failure.
 977 */
 978static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan)
 979{
 980        union ipu_channel_param *params = &ichan->params;
 981        uint32_t ipu_conf;
 982        enum ipu_channel channel = ichan->dma_chan.chan_id;
 983        unsigned long flags;
 984        uint32_t reg;
 985        struct ipu *ipu = to_ipu(idmac);
 986        int ret = 0, n_desc = 0;
 987
 988        dev_dbg(ipu->dev, "init channel = %d\n", channel);
 989
 990        if (channel != IDMAC_SDC_0 && channel != IDMAC_SDC_1 &&
 991            channel != IDMAC_IC_7)
 992                return -EINVAL;
 993
 994        spin_lock_irqsave(&ipu->lock, flags);
 995
 996        switch (channel) {
 997        case IDMAC_IC_7:
 998                n_desc = 16;
 999                reg = idmac_read_icreg(ipu, IC_CONF);
1000                idmac_write_icreg(ipu, reg & ~IC_CONF_CSI_MEM_WR_EN, IC_CONF);
1001                break;
1002        case IDMAC_IC_0:
1003                n_desc = 16;
1004                reg = idmac_read_ipureg(ipu, IPU_FS_PROC_FLOW);
1005                idmac_write_ipureg(ipu, reg & ~FS_ENC_IN_VALID, IPU_FS_PROC_FLOW);
1006                ret = ipu_ic_init_prpenc(ipu, params, true);
1007                break;
1008        case IDMAC_SDC_0:
1009        case IDMAC_SDC_1:
1010                n_desc = 4;
1011        default:
1012                break;
1013        }
1014
1015        ipu->channel_init_mask |= 1L << channel;
1016
1017        /* Enable IPU sub module */
1018        ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) |
1019                ipu_channel_conf_mask(channel);
1020        idmac_write_ipureg(ipu, ipu_conf, IPU_CONF);
1021
1022        spin_unlock_irqrestore(&ipu->lock, flags);
1023
1024        if (n_desc && !ichan->desc)
1025                ret = idmac_desc_alloc(ichan, n_desc);
1026
1027        dump_idmac_reg(ipu);
1028
1029        return ret;
1030}
1031
1032/**
1033 * ipu_uninit_channel() - uninitialize an IPU channel.
1034 * @idmac:      IPU DMAC context.
1035 * @ichan:      pointer to the channel object.
1036 */
1037static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan)
1038{
1039        enum ipu_channel channel = ichan->dma_chan.chan_id;
1040        unsigned long flags;
1041        uint32_t reg;
1042        unsigned long chan_mask = 1UL << channel;
1043        uint32_t ipu_conf;
1044        struct ipu *ipu = to_ipu(idmac);
1045
1046        spin_lock_irqsave(&ipu->lock, flags);
1047
1048        if (!(ipu->channel_init_mask & chan_mask)) {
1049                dev_err(ipu->dev, "Channel already uninitialized %d\n",
1050                        channel);
1051                spin_unlock_irqrestore(&ipu->lock, flags);
1052                return;
1053        }
1054
1055        /* Reset the double buffer */
1056        reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL);
1057        idmac_write_ipureg(ipu, reg & ~chan_mask, IPU_CHA_DB_MODE_SEL);
1058
1059        ichan->sec_chan_en = false;
1060
1061        switch (channel) {
1062        case IDMAC_IC_7:
1063                reg = idmac_read_icreg(ipu, IC_CONF);
1064                idmac_write_icreg(ipu, reg & ~(IC_CONF_RWS_EN | IC_CONF_PRPENC_EN),
1065                             IC_CONF);
1066                break;
1067        case IDMAC_IC_0:
1068                reg = idmac_read_icreg(ipu, IC_CONF);
1069                idmac_write_icreg(ipu, reg & ~(IC_CONF_PRPENC_EN | IC_CONF_PRPENC_CSC1),
1070                                  IC_CONF);
1071                break;
1072        case IDMAC_SDC_0:
1073        case IDMAC_SDC_1:
1074        default:
1075                break;
1076        }
1077
1078        ipu->channel_init_mask &= ~(1L << channel);
1079
1080        ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) &
1081                ~ipu_channel_conf_mask(channel);
1082        idmac_write_ipureg(ipu, ipu_conf, IPU_CONF);
1083
1084        spin_unlock_irqrestore(&ipu->lock, flags);
1085
1086        ichan->n_tx_desc = 0;
1087        vfree(ichan->desc);
1088        ichan->desc = NULL;
1089}
1090
1091/**
1092 * ipu_disable_channel() - disable an IPU channel.
1093 * @idmac:              IPU DMAC context.
1094 * @ichan:              channel object pointer.
1095 * @wait_for_stop:      flag to set whether to wait for channel end of frame or
1096 *                      return immediately.
1097 * @return:             0 on success or negative error code on failure.
1098 */
1099static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
1100                               bool wait_for_stop)
1101{
1102        enum ipu_channel channel = ichan->dma_chan.chan_id;
1103        struct ipu *ipu = to_ipu(idmac);
1104        uint32_t reg;
1105        unsigned long flags;
1106        unsigned long chan_mask = 1UL << channel;
1107        unsigned int timeout;
1108
1109        if (wait_for_stop && channel != IDMAC_SDC_1 && channel != IDMAC_SDC_0) {
1110                timeout = 40;
1111                /* This waiting always fails. Related to spurious irq problem */
1112                while ((idmac_read_icreg(ipu, IDMAC_CHA_BUSY) & chan_mask) ||
1113                       (ipu_channel_status(ipu, channel) == TASK_STAT_ACTIVE)) {
1114                        timeout--;
1115                        msleep(10);
1116
1117                        if (!timeout) {
1118                                dev_dbg(ipu->dev,
1119                                        "Warning: timeout waiting for channel %u to "
1120                                        "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
1121                                        "busy = 0x%08X, tstat = 0x%08X\n", channel,
1122                                        idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY),
1123                                        idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY),
1124                                        idmac_read_icreg(ipu, IDMAC_CHA_BUSY),
1125                                        idmac_read_ipureg(ipu, IPU_TASKS_STAT));
1126                                break;
1127                        }
1128                }
1129                dev_dbg(ipu->dev, "timeout = %d * 10ms\n", 40 - timeout);
1130        }
1131        /* SDC BG and FG must be disabled before DMA is disabled */
1132        if (wait_for_stop && (channel == IDMAC_SDC_0 ||
1133                              channel == IDMAC_SDC_1)) {
1134                for (timeout = 5;
1135                     timeout && !ipu_irq_status(ichan->eof_irq); timeout--)
1136                        msleep(5);
1137        }
1138
1139        spin_lock_irqsave(&ipu->lock, flags);
1140
1141        /* Disable IC task */
1142        ipu_ic_disable_task(ipu, channel);
1143
1144        /* Disable DMA channel(s) */
1145        reg = idmac_read_icreg(ipu, IDMAC_CHA_EN);
1146        idmac_write_icreg(ipu, reg & ~chan_mask, IDMAC_CHA_EN);
1147
1148        spin_unlock_irqrestore(&ipu->lock, flags);
1149
1150        return 0;
1151}
1152
1153static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan,
1154        struct idmac_tx_desc **desc, struct scatterlist *sg)
1155{
1156        struct scatterlist *sgnew = sg ? sg_next(sg) : NULL;
1157
1158        if (sgnew)
1159                /* next sg-element in this list */
1160                return sgnew;
1161
1162        if ((*desc)->list.next == &ichan->queue)
1163                /* No more descriptors on the queue */
1164                return NULL;
1165
1166        /* Fetch next descriptor */
1167        *desc = list_entry((*desc)->list.next, struct idmac_tx_desc, list);
1168        return (*desc)->sg;
1169}
1170
1171/*
1172 * We have several possibilities here:
1173 * current BUF          next BUF
1174 *
1175 * not last sg          next not last sg
1176 * not last sg          next last sg
1177 * last sg              first sg from next descriptor
1178 * last sg              NULL
1179 *
1180 * Besides, the descriptor queue might be empty or not. We process all these
1181 * cases carefully.
1182 */
1183static irqreturn_t idmac_interrupt(int irq, void *dev_id)
1184{
1185        struct idmac_channel *ichan = dev_id;
1186        struct device *dev = &ichan->dma_chan.dev->device;
1187        unsigned int chan_id = ichan->dma_chan.chan_id;
1188        struct scatterlist **sg, *sgnext, *sgnew = NULL;
1189        /* Next transfer descriptor */
1190        struct idmac_tx_desc *desc, *descnew;
1191        dma_async_tx_callback callback;
1192        void *callback_param;
1193        bool done = false;
1194        u32 ready0, ready1, curbuf, err;
1195        unsigned long flags;
1196
1197        /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
1198
1199        dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);
1200
1201        spin_lock_irqsave(&ipu_data.lock, flags);
1202
1203        ready0  = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
1204        ready1  = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
1205        curbuf  = idmac_read_ipureg(&ipu_data, IPU_CHA_CUR_BUF);
1206        err     = idmac_read_ipureg(&ipu_data, IPU_INT_STAT_4);
1207
1208        if (err & (1 << chan_id)) {
1209                idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4);
1210                spin_unlock_irqrestore(&ipu_data.lock, flags);
1211                /*
1212                 * Doing this
1213                 * ichan->sg[0] = ichan->sg[1] = NULL;
1214                 * you can force channel re-enable on the next tx_submit(), but
1215                 * this is dirty - think about descriptors with multiple
1216                 * sg elements.
1217                 */
1218                dev_warn(dev, "NFB4EOF on channel %d, ready %x, %x, cur %x\n",
1219                         chan_id, ready0, ready1, curbuf);
1220                return IRQ_HANDLED;
1221        }
1222        spin_unlock_irqrestore(&ipu_data.lock, flags);
1223
1224        /* Other interrupts do not interfere with this channel */
1225        spin_lock(&ichan->lock);
1226        if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) ||
1227                     (!ichan->active_buffer && (ready0 >> chan_id) & 1)
1228                     )) {
1229                spin_unlock(&ichan->lock);
1230                dev_dbg(dev,
1231                        "IRQ with active buffer still ready on channel %x, "
1232                        "active %d, ready %x, %x!\n", chan_id,
1233                        ichan->active_buffer, ready0, ready1);
1234                return IRQ_NONE;
1235        }
1236
1237        if (unlikely(list_empty(&ichan->queue))) {
1238                ichan->sg[ichan->active_buffer] = NULL;
1239                spin_unlock(&ichan->lock);
1240                dev_err(dev,
1241                        "IRQ without queued buffers on channel %x, active %d, "
1242                        "ready %x, %x!\n", chan_id,
1243                        ichan->active_buffer, ready0, ready1);
1244                return IRQ_NONE;
1245        }
1246
1247        /*
1248         * active_buffer is a software flag, it shows which buffer we are
1249         * currently expecting back from the hardware, IDMAC should be
1250         * processing the other buffer already
1251         */
1252        sg = &ichan->sg[ichan->active_buffer];
1253        sgnext = ichan->sg[!ichan->active_buffer];
1254
1255        if (!*sg) {
1256                spin_unlock(&ichan->lock);
1257                return IRQ_HANDLED;
1258        }
1259
1260        desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list);
1261        descnew = desc;
1262
1263        dev_dbg(dev, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n",
1264                irq, sg_dma_address(*sg), sgnext ? sg_dma_address(sgnext) : 0, ichan->active_buffer, curbuf);
1265
1266        /* Find the descriptor of sgnext */
1267        sgnew = idmac_sg_next(ichan, &descnew, *sg);
1268        if (sgnext != sgnew)
1269                dev_err(dev, "Submitted buffer %p, next buffer %p\n", sgnext, sgnew);
1270
1271        /*
1272         * if sgnext == NULL sg must be the last element in a scatterlist and
1273         * queue must be empty
1274         */
1275        if (unlikely(!sgnext)) {
1276                if (!WARN_ON(sg_next(*sg)))
1277                        dev_dbg(dev, "Underrun on channel %x\n", chan_id);
1278                ichan->sg[!ichan->active_buffer] = sgnew;
1279
1280                if (unlikely(sgnew)) {
1281                        ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);
1282                } else {
1283                        spin_lock_irqsave(&ipu_data.lock, flags);
1284                        ipu_ic_disable_task(&ipu_data, chan_id);
1285                        spin_unlock_irqrestore(&ipu_data.lock, flags);
1286                        ichan->status = IPU_CHANNEL_READY;
1287                        /* Continue to check for complete descriptor */
1288                }
1289        }
1290
1291        /* Calculate and submit the next sg element */
1292        sgnew = idmac_sg_next(ichan, &descnew, sgnew);
1293
1294        if (unlikely(!sg_next(*sg)) || !sgnext) {
1295                /*
1296                 * Last element in scatterlist done, remove from the queue,
1297                 * _init for debugging
1298                 */
1299                list_del_init(&desc->list);
1300                done = true;
1301        }
1302
1303        *sg = sgnew;
1304
1305        if (likely(sgnew) &&
1306            ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {
1307                callback = descnew->txd.callback;
1308                callback_param = descnew->txd.callback_param;
1309                spin_unlock(&ichan->lock);
1310                if (callback)
1311                        callback(callback_param);
1312                spin_lock(&ichan->lock);
1313        }
1314
1315        /* Flip the active buffer - even if update above failed */
1316        ichan->active_buffer = !ichan->active_buffer;
1317        if (done)
1318                ichan->completed = desc->txd.cookie;
1319
1320        callback = desc->txd.callback;
1321        callback_param = desc->txd.callback_param;
1322
1323        spin_unlock(&ichan->lock);
1324
1325        if (done && (desc->txd.flags & DMA_PREP_INTERRUPT) && callback)
1326                callback(callback_param);
1327
1328        return IRQ_HANDLED;
1329}
1330
1331static void ipu_gc_tasklet(unsigned long arg)
1332{
1333        struct ipu *ipu = (struct ipu *)arg;
1334        int i;
1335
1336        for (i = 0; i < IPU_CHANNELS_NUM; i++) {
1337                struct idmac_channel *ichan = ipu->channel + i;
1338                struct idmac_tx_desc *desc;
1339                unsigned long flags;
1340                struct scatterlist *sg;
1341                int j, k;
1342
1343                for (j = 0; j < ichan->n_tx_desc; j++) {
1344                        desc = ichan->desc + j;
1345                        spin_lock_irqsave(&ichan->lock, flags);
1346                        if (async_tx_test_ack(&desc->txd)) {
1347                                list_move(&desc->list, &ichan->free_list);
1348                                for_each_sg(desc->sg, sg, desc->sg_len, k) {
1349                                        if (ichan->sg[0] == sg)
1350                                                ichan->sg[0] = NULL;
1351                                        else if (ichan->sg[1] == sg)
1352                                                ichan->sg[1] = NULL;
1353                                }
1354                                async_tx_clear_ack(&desc->txd);
1355                        }
1356                        spin_unlock_irqrestore(&ichan->lock, flags);
1357                }
1358        }
1359}
1360
1361/* Allocate and initialise a transfer descriptor. */
1362static struct dma_async_tx_descriptor *idmac_prep_slave_sg(struct dma_chan *chan,
1363                struct scatterlist *sgl, unsigned int sg_len,
1364                enum dma_data_direction direction, unsigned long tx_flags)
1365{
1366        struct idmac_channel *ichan = to_idmac_chan(chan);
1367        struct idmac_tx_desc *desc = NULL;
1368        struct dma_async_tx_descriptor *txd = NULL;
1369        unsigned long flags;
1370
1371        /* We only can handle these three channels so far */
1372        if (chan->chan_id != IDMAC_SDC_0 && chan->chan_id != IDMAC_SDC_1 &&
1373            chan->chan_id != IDMAC_IC_7)
1374                return NULL;
1375
1376        if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE) {
1377                dev_err(chan->device->dev, "Invalid DMA direction %d!\n", direction);
1378                return NULL;
1379        }
1380
1381        mutex_lock(&ichan->chan_mutex);
1382
1383        spin_lock_irqsave(&ichan->lock, flags);
1384        if (!list_empty(&ichan->free_list)) {
1385                desc = list_entry(ichan->free_list.next,
1386                                  struct idmac_tx_desc, list);
1387
1388                list_del_init(&desc->list);
1389
1390                desc->sg_len    = sg_len;
1391                desc->sg        = sgl;
1392                txd             = &desc->txd;
1393                txd->flags      = tx_flags;
1394        }
1395        spin_unlock_irqrestore(&ichan->lock, flags);
1396
1397        mutex_unlock(&ichan->chan_mutex);
1398
1399        tasklet_schedule(&to_ipu(to_idmac(chan->device))->tasklet);
1400
1401        return txd;
1402}
1403
1404/* Re-select the current buffer and re-activate the channel */
1405static void idmac_issue_pending(struct dma_chan *chan)
1406{
1407        struct idmac_channel *ichan = to_idmac_chan(chan);
1408        struct idmac *idmac = to_idmac(chan->device);
1409        struct ipu *ipu = to_ipu(idmac);
1410        unsigned long flags;
1411
1412        /* This is not always needed, but doesn't hurt either */
1413        spin_lock_irqsave(&ipu->lock, flags);
1414        ipu_select_buffer(chan->chan_id, ichan->active_buffer);
1415        spin_unlock_irqrestore(&ipu->lock, flags);
1416
1417        /*
1418         * Might need to perform some parts of initialisation from
1419         * ipu_enable_channel(), but not all, we do not want to reset to buffer
1420         * 0, don't need to set priority again either, but re-enabling the task
1421         * and the channel might be a good idea.
1422         */
1423}
1424
1425static int __idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1426                           unsigned long arg)
1427{
1428        struct idmac_channel *ichan = to_idmac_chan(chan);
1429        struct idmac *idmac = to_idmac(chan->device);
1430        unsigned long flags;
1431        int i;
1432
1433        /* Only supports DMA_TERMINATE_ALL */
1434        if (cmd != DMA_TERMINATE_ALL)
1435                return -ENXIO;
1436
1437        ipu_disable_channel(idmac, ichan,
1438                            ichan->status >= IPU_CHANNEL_ENABLED);
1439
1440        tasklet_disable(&to_ipu(idmac)->tasklet);
1441
1442        /* ichan->queue is modified in ISR, have to spinlock */
1443        spin_lock_irqsave(&ichan->lock, flags);
1444        list_splice_init(&ichan->queue, &ichan->free_list);
1445
1446        if (ichan->desc)
1447                for (i = 0; i < ichan->n_tx_desc; i++) {
1448                        struct idmac_tx_desc *desc = ichan->desc + i;
1449                        if (list_empty(&desc->list))
1450                                /* Descriptor was prepared, but not submitted */
1451                                list_add(&desc->list, &ichan->free_list);
1452
1453                        async_tx_clear_ack(&desc->txd);
1454                }
1455
1456        ichan->sg[0] = NULL;
1457        ichan->sg[1] = NULL;
1458        spin_unlock_irqrestore(&ichan->lock, flags);
1459
1460        tasklet_enable(&to_ipu(idmac)->tasklet);
1461
1462        ichan->status = IPU_CHANNEL_INITIALIZED;
1463
1464        return 0;
1465}
1466
1467static int idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1468                         unsigned long arg)
1469{
1470        struct idmac_channel *ichan = to_idmac_chan(chan);
1471        int ret;
1472
1473        mutex_lock(&ichan->chan_mutex);
1474
1475        ret = __idmac_control(chan, cmd, arg);
1476
1477        mutex_unlock(&ichan->chan_mutex);
1478
1479        return ret;
1480}
1481
1482#ifdef DEBUG
1483static irqreturn_t ic_sof_irq(int irq, void *dev_id)
1484{
1485        struct idmac_channel *ichan = dev_id;
1486        printk(KERN_DEBUG "Got SOF IRQ %d on Channel %d\n",
1487               irq, ichan->dma_chan.chan_id);
1488        disable_irq_nosync(irq);
1489        return IRQ_HANDLED;
1490}
1491
1492static irqreturn_t ic_eof_irq(int irq, void *dev_id)
1493{
1494        struct idmac_channel *ichan = dev_id;
1495        printk(KERN_DEBUG "Got EOF IRQ %d on Channel %d\n",
1496               irq, ichan->dma_chan.chan_id);
1497        disable_irq_nosync(irq);
1498        return IRQ_HANDLED;
1499}
1500
1501static int ic_sof = -EINVAL, ic_eof = -EINVAL;
1502#endif
1503
1504static int idmac_alloc_chan_resources(struct dma_chan *chan)
1505{
1506        struct idmac_channel *ichan = to_idmac_chan(chan);
1507        struct idmac *idmac = to_idmac(chan->device);
1508        int ret;
1509
1510        /* dmaengine.c now guarantees to only offer free channels */
1511        BUG_ON(chan->client_count > 1);
1512        WARN_ON(ichan->status != IPU_CHANNEL_FREE);
1513
1514        chan->cookie            = 1;
1515        ichan->completed        = -ENXIO;
1516
1517        ret = ipu_irq_map(chan->chan_id);
1518        if (ret < 0)
1519                goto eimap;
1520
1521        ichan->eof_irq = ret;
1522
1523        /*
1524         * Important to first disable the channel, because maybe someone
1525         * used it before us, e.g., the bootloader
1526         */
1527        ipu_disable_channel(idmac, ichan, true);
1528
1529        ret = ipu_init_channel(idmac, ichan);
1530        if (ret < 0)
1531                goto eichan;
1532
1533        ret = request_irq(ichan->eof_irq, idmac_interrupt, 0,
1534                          ichan->eof_name, ichan);
1535        if (ret < 0)
1536                goto erirq;
1537
1538#ifdef DEBUG
1539        if (chan->chan_id == IDMAC_IC_7) {
1540                ic_sof = ipu_irq_map(69);
1541                if (ic_sof > 0)
1542                        request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan);
1543                ic_eof = ipu_irq_map(70);
1544                if (ic_eof > 0)
1545                        request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan);
1546        }
1547#endif
1548
1549        ichan->status = IPU_CHANNEL_INITIALIZED;
1550
1551        dev_dbg(&chan->dev->device, "Found channel 0x%x, irq %d\n",
1552                chan->chan_id, ichan->eof_irq);
1553
1554        return ret;
1555
1556erirq:
1557        ipu_uninit_channel(idmac, ichan);
1558eichan:
1559        ipu_irq_unmap(chan->chan_id);
1560eimap:
1561        return ret;
1562}
1563
1564static void idmac_free_chan_resources(struct dma_chan *chan)
1565{
1566        struct idmac_channel *ichan = to_idmac_chan(chan);
1567        struct idmac *idmac = to_idmac(chan->device);
1568
1569        mutex_lock(&ichan->chan_mutex);
1570
1571        __idmac_control(chan, DMA_TERMINATE_ALL, 0);
1572
1573        if (ichan->status > IPU_CHANNEL_FREE) {
1574#ifdef DEBUG
1575                if (chan->chan_id == IDMAC_IC_7) {
1576                        if (ic_sof > 0) {
1577                                free_irq(ic_sof, ichan);
1578                                ipu_irq_unmap(69);
1579                                ic_sof = -EINVAL;
1580                        }
1581                        if (ic_eof > 0) {
1582                                free_irq(ic_eof, ichan);
1583                                ipu_irq_unmap(70);
1584                                ic_eof = -EINVAL;
1585                        }
1586                }
1587#endif
1588                free_irq(ichan->eof_irq, ichan);
1589                ipu_irq_unmap(chan->chan_id);
1590        }
1591
1592        ichan->status = IPU_CHANNEL_FREE;
1593
1594        ipu_uninit_channel(idmac, ichan);
1595
1596        mutex_unlock(&ichan->chan_mutex);
1597
1598        tasklet_schedule(&to_ipu(idmac)->tasklet);
1599}
1600
1601static enum dma_status idmac_tx_status(struct dma_chan *chan,
1602                       dma_cookie_t cookie, struct dma_tx_state *txstate)
1603{
1604        struct idmac_channel *ichan = to_idmac_chan(chan);
1605
1606        dma_set_tx_state(txstate, ichan->completed, chan->cookie, 0);
1607        if (cookie != chan->cookie)
1608                return DMA_ERROR;
1609        return DMA_SUCCESS;
1610}
1611
1612static int __init ipu_idmac_init(struct ipu *ipu)
1613{
1614        struct idmac *idmac = &ipu->idmac;
1615        struct dma_device *dma = &idmac->dma;
1616        int i;
1617
1618        dma_cap_set(DMA_SLAVE, dma->cap_mask);
1619        dma_cap_set(DMA_PRIVATE, dma->cap_mask);
1620
1621        /* Compulsory common fields */
1622        dma->dev                                = ipu->dev;
1623        dma->device_alloc_chan_resources        = idmac_alloc_chan_resources;
1624        dma->device_free_chan_resources         = idmac_free_chan_resources;
1625        dma->device_tx_status                   = idmac_tx_status;
1626        dma->device_issue_pending               = idmac_issue_pending;
1627
1628        /* Compulsory for DMA_SLAVE fields */
1629        dma->device_prep_slave_sg               = idmac_prep_slave_sg;
1630        dma->device_control                     = idmac_control;
1631
1632        INIT_LIST_HEAD(&dma->channels);
1633        for (i = 0; i < IPU_CHANNELS_NUM; i++) {
1634                struct idmac_channel *ichan = ipu->channel + i;
1635                struct dma_chan *dma_chan = &ichan->dma_chan;
1636
1637                spin_lock_init(&ichan->lock);
1638                mutex_init(&ichan->chan_mutex);
1639
1640                ichan->status           = IPU_CHANNEL_FREE;
1641                ichan->sec_chan_en      = false;
1642                ichan->completed        = -ENXIO;
1643                snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i);
1644
1645                dma_chan->device        = &idmac->dma;
1646                dma_chan->cookie        = 1;
1647                dma_chan->chan_id       = i;
1648                list_add_tail(&dma_chan->device_node, &dma->channels);
1649        }
1650
1651        idmac_write_icreg(ipu, 0x00000070, IDMAC_CONF);
1652
1653        return dma_async_device_register(&idmac->dma);
1654}
1655
1656static void __exit ipu_idmac_exit(struct ipu *ipu)
1657{
1658        int i;
1659        struct idmac *idmac = &ipu->idmac;
1660
1661        for (i = 0; i < IPU_CHANNELS_NUM; i++) {
1662                struct idmac_channel *ichan = ipu->channel + i;
1663
1664                idmac_control(&ichan->dma_chan, DMA_TERMINATE_ALL, 0);
1665                idmac_prep_slave_sg(&ichan->dma_chan, NULL, 0, DMA_NONE, 0);
1666        }
1667
1668        dma_async_device_unregister(&idmac->dma);
1669}
1670
1671/*****************************************************************************
1672 * IPU common probe / remove
1673 */
1674
1675static int __init ipu_probe(struct platform_device *pdev)
1676{
1677        struct ipu_platform_data *pdata = pdev->dev.platform_data;
1678        struct resource *mem_ipu, *mem_ic;
1679        int ret;
1680
1681        spin_lock_init(&ipu_data.lock);
1682
1683        mem_ipu = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1684        mem_ic  = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1685        if (!pdata || !mem_ipu || !mem_ic)
1686                return -EINVAL;
1687
1688        ipu_data.dev = &pdev->dev;
1689
1690        platform_set_drvdata(pdev, &ipu_data);
1691
1692        ret = platform_get_irq(pdev, 0);
1693        if (ret < 0)
1694                goto err_noirq;
1695
1696        ipu_data.irq_fn = ret;
1697        ret = platform_get_irq(pdev, 1);
1698        if (ret < 0)
1699                goto err_noirq;
1700
1701        ipu_data.irq_err = ret;
1702        ipu_data.irq_base = pdata->irq_base;
1703
1704        dev_dbg(&pdev->dev, "fn irq %u, err irq %u, irq-base %u\n",
1705                ipu_data.irq_fn, ipu_data.irq_err, ipu_data.irq_base);
1706
1707        /* Remap IPU common registers */
1708        ipu_data.reg_ipu = ioremap(mem_ipu->start,
1709                                   mem_ipu->end - mem_ipu->start + 1);
1710        if (!ipu_data.reg_ipu) {
1711                ret = -ENOMEM;
1712                goto err_ioremap_ipu;
1713        }
1714
1715        /* Remap Image Converter and Image DMA Controller registers */
1716        ipu_data.reg_ic = ioremap(mem_ic->start,
1717                                  mem_ic->end - mem_ic->start + 1);
1718        if (!ipu_data.reg_ic) {
1719                ret = -ENOMEM;
1720                goto err_ioremap_ic;
1721        }
1722
1723        /* Get IPU clock */
1724        ipu_data.ipu_clk = clk_get(&pdev->dev, NULL);
1725        if (IS_ERR(ipu_data.ipu_clk)) {
1726                ret = PTR_ERR(ipu_data.ipu_clk);
1727                goto err_clk_get;
1728        }
1729
1730        /* Make sure IPU HSP clock is running */
1731        clk_enable(ipu_data.ipu_clk);
1732
1733        /* Disable all interrupts */
1734        idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_1);
1735        idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_2);
1736        idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_3);
1737        idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_4);
1738        idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_5);
1739
1740        dev_dbg(&pdev->dev, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev->name,
1741                (unsigned long)mem_ipu->start, ipu_data.irq_fn, ipu_data.irq_err);
1742
1743        ret = ipu_irq_attach_irq(&ipu_data, pdev);
1744        if (ret < 0)
1745                goto err_attach_irq;
1746
1747        /* Initialize DMA engine */
1748        ret = ipu_idmac_init(&ipu_data);
1749        if (ret < 0)
1750                goto err_idmac_init;
1751
1752        tasklet_init(&ipu_data.tasklet, ipu_gc_tasklet, (unsigned long)&ipu_data);
1753
1754        ipu_data.dev = &pdev->dev;
1755
1756        dev_dbg(ipu_data.dev, "IPU initialized\n");
1757
1758        return 0;
1759
1760err_idmac_init:
1761err_attach_irq:
1762        ipu_irq_detach_irq(&ipu_data, pdev);
1763        clk_disable(ipu_data.ipu_clk);
1764        clk_put(ipu_data.ipu_clk);
1765err_clk_get:
1766        iounmap(ipu_data.reg_ic);
1767err_ioremap_ic:
1768        iounmap(ipu_data.reg_ipu);
1769err_ioremap_ipu:
1770err_noirq:
1771        dev_err(&pdev->dev, "Failed to probe IPU: %d\n", ret);
1772        return ret;
1773}
1774
1775static int __exit ipu_remove(struct platform_device *pdev)
1776{
1777        struct ipu *ipu = platform_get_drvdata(pdev);
1778
1779        ipu_idmac_exit(ipu);
1780        ipu_irq_detach_irq(ipu, pdev);
1781        clk_disable(ipu->ipu_clk);
1782        clk_put(ipu->ipu_clk);
1783        iounmap(ipu->reg_ic);
1784        iounmap(ipu->reg_ipu);
1785        tasklet_kill(&ipu->tasklet);
1786        platform_set_drvdata(pdev, NULL);
1787
1788        return 0;
1789}
1790
1791/*
1792 * We need two MEM resources - with IPU-common and Image Converter registers,
1793 * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
1794 */
1795static struct platform_driver ipu_platform_driver = {
1796        .driver = {
1797                .name   = "ipu-core",
1798                .owner  = THIS_MODULE,
1799        },
1800        .remove         = __exit_p(ipu_remove),
1801};
1802
1803static int __init ipu_init(void)
1804{
1805        return platform_driver_probe(&ipu_platform_driver, ipu_probe);
1806}
1807subsys_initcall(ipu_init);
1808
1809MODULE_DESCRIPTION("IPU core driver");
1810MODULE_LICENSE("GPL v2");
1811MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
1812MODULE_ALIAS("platform:ipu-core");
1813