linux/drivers/staging/sep/sep_main.c
<<
>>
Prefs
   1/*
   2 *
   3 *  sep_main.c - Security Processor Driver main group of functions
   4 *
   5 *  Copyright(c) 2009-2011 Intel Corporation. All rights reserved.
   6 *  Contributions(c) 2009-2011 Discretix. All rights reserved.
   7 *
   8 *  This program is free software; you can redistribute it and/or modify it
   9 *  under the terms of the GNU General Public License as published by the Free
  10 *  Software Foundation; version 2 of the License.
  11 *
  12 *  This program is distributed in the hope that it will be useful, but WITHOUT
  13 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15 *  more details.
  16 *
  17 *  You should have received a copy of the GNU General Public License along with
  18 *  this program; if not, write to the Free Software Foundation, Inc., 59
  19 *  Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  20 *
  21 *  CONTACTS:
  22 *
  23 *  Mark Allyn          mark.a.allyn@intel.com
  24 *  Jayant Mangalampalli jayant.mangalampalli@intel.com
  25 *
  26 *  CHANGES:
  27 *
  28 *  2009.06.26  Initial publish
  29 *  2010.09.14  Upgrade to Medfield
  30 *  2011.01.21  Move to sep_main.c to allow for sep_crypto.c
  31 *  2011.02.22  Enable kernel crypto operation
  32 *
  33 *  Please note that this driver is based on information in the Discretix
  34 *  CryptoCell 5.2 Driver Implementation Guide; the Discretix CryptoCell 5.2
  35 *  Integration Intel Medfield appendix; the Discretix CryptoCell 5.2
  36 *  Linux Driver Integration Guide; and the Discretix CryptoCell 5.2 System
  37 *  Overview and Integration Guide.
  38 */
  39/* #define DEBUG */
  40/* #define SEP_PERF_DEBUG */
  41
  42#include <linux/kernel.h>
  43#include <linux/module.h>
  44#include <linux/miscdevice.h>
  45#include <linux/fs.h>
  46#include <linux/cdev.h>
  47#include <linux/kdev_t.h>
  48#include <linux/mutex.h>
  49#include <linux/sched.h>
  50#include <linux/mm.h>
  51#include <linux/poll.h>
  52#include <linux/wait.h>
  53#include <linux/pci.h>
  54#include <linux/pm_runtime.h>
  55#include <linux/slab.h>
  56#include <linux/ioctl.h>
  57#include <asm/current.h>
  58#include <linux/ioport.h>
  59#include <linux/io.h>
  60#include <linux/interrupt.h>
  61#include <linux/pagemap.h>
  62#include <asm/cacheflush.h>
  63#include <linux/delay.h>
  64#include <linux/jiffies.h>
  65#include <linux/async.h>
  66#include <linux/crypto.h>
  67#include <crypto/internal/hash.h>
  68#include <crypto/scatterwalk.h>
  69#include <crypto/sha.h>
  70#include <crypto/md5.h>
  71#include <crypto/aes.h>
  72#include <crypto/des.h>
  73#include <crypto/hash.h>
  74
  75#include "sep_driver_hw_defs.h"
  76#include "sep_driver_config.h"
  77#include "sep_driver_api.h"
  78#include "sep_dev.h"
  79#include "sep_crypto.h"
  80
  81#define CREATE_TRACE_POINTS
  82#include "sep_trace_events.h"
  83
  84/*
  85 * Let's not spend cycles iterating over message
  86 * area contents if debugging not enabled
  87 */
  88#ifdef DEBUG
  89#define sep_dump_message(sep)   _sep_dump_message(sep)
  90#else
  91#define sep_dump_message(sep)
  92#endif
  93
  94/**
  95 * Currently, there is only one SEP device per platform;
  96 * In event platforms in the future have more than one SEP
  97 * device, this will be a linked list
  98 */
  99
 100struct sep_device *sep_dev;
 101
 102/**
 103 * sep_queue_status_remove - Removes transaction from status queue
 104 * @sep: SEP device
 105 * @sep_queue_info: pointer to status queue
 106 *
 107 * This function will remove information about transaction from the queue.
 108 */
 109void sep_queue_status_remove(struct sep_device *sep,
 110                                      struct sep_queue_info **queue_elem)
 111{
 112        unsigned long lck_flags;
 113
 114        dev_dbg(&sep->pdev->dev, "[PID%d] sep_queue_status_remove\n",
 115                current->pid);
 116
 117        if (!queue_elem || !(*queue_elem)) {
 118                dev_dbg(&sep->pdev->dev, "PID%d %s null\n",
 119                                        current->pid, __func__);
 120                return;
 121        }
 122
 123        spin_lock_irqsave(&sep->sep_queue_lock, lck_flags);
 124        list_del(&(*queue_elem)->list);
 125        sep->sep_queue_num--;
 126        spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags);
 127
 128        kfree(*queue_elem);
 129        *queue_elem = NULL;
 130
 131        dev_dbg(&sep->pdev->dev, "[PID%d] sep_queue_status_remove return\n",
 132                current->pid);
 133        return;
 134}
 135
 136/**
 137 * sep_queue_status_add - Adds transaction to status queue
 138 * @sep: SEP device
 139 * @opcode: transaction opcode
 140 * @size: input data size
 141 * @pid: pid of current process
 142 * @name: current process name
 143 * @name_len: length of name (current process)
 144 *
 145 * This function adds information about about transaction started to the status
 146 * queue.
 147 */
 148struct sep_queue_info *sep_queue_status_add(
 149                                                struct sep_device *sep,
 150                                                u32 opcode,
 151                                                u32 size,
 152                                                u32 pid,
 153                                                u8 *name, size_t name_len)
 154{
 155        unsigned long lck_flags;
 156        struct sep_queue_info *my_elem = NULL;
 157
 158        my_elem = kzalloc(sizeof(struct sep_queue_info), GFP_KERNEL);
 159
 160        if (!my_elem)
 161                return NULL;
 162
 163        dev_dbg(&sep->pdev->dev, "[PID%d] kzalloc ok\n", current->pid);
 164
 165        my_elem->data.opcode = opcode;
 166        my_elem->data.size = size;
 167        my_elem->data.pid = pid;
 168
 169        if (name_len > TASK_COMM_LEN)
 170                name_len = TASK_COMM_LEN;
 171
 172        memcpy(&my_elem->data.name, name, name_len);
 173
 174        spin_lock_irqsave(&sep->sep_queue_lock, lck_flags);
 175
 176        list_add_tail(&my_elem->list, &sep->sep_queue_status);
 177        sep->sep_queue_num++;
 178
 179        spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags);
 180
 181        return my_elem;
 182}
 183
 184/**
 185 *      sep_allocate_dmatables_region - Allocates buf for the MLLI/DMA tables
 186 *      @sep: SEP device
 187 *      @dmatables_region: Destination pointer for the buffer
 188 *      @dma_ctx: DMA context for the transaction
 189 *      @table_count: Number of MLLI/DMA tables to create
 190 *      The buffer created will not work as-is for DMA operations,
 191 *      it needs to be copied over to the appropriate place in the
 192 *      shared area.
 193 */
 194static int sep_allocate_dmatables_region(struct sep_device *sep,
 195                                         void **dmatables_region,
 196                                         struct sep_dma_context *dma_ctx,
 197                                         const u32 table_count)
 198{
 199        const size_t new_len =
 200                SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES - 1;
 201
 202        void *tmp_region = NULL;
 203
 204        dev_dbg(&sep->pdev->dev, "[PID%d] dma_ctx = 0x%p\n",
 205                                current->pid, dma_ctx);
 206        dev_dbg(&sep->pdev->dev, "[PID%d] dmatables_region = 0x%p\n",
 207                                current->pid, dmatables_region);
 208
 209        if (!dma_ctx || !dmatables_region) {
 210                dev_warn(&sep->pdev->dev,
 211                        "[PID%d] dma context/region uninitialized\n",
 212                        current->pid);
 213                return -EINVAL;
 214        }
 215
 216        dev_dbg(&sep->pdev->dev, "[PID%d] newlen = 0x%08zX\n",
 217                                current->pid, new_len);
 218        dev_dbg(&sep->pdev->dev, "[PID%d] oldlen = 0x%08X\n", current->pid,
 219                                dma_ctx->dmatables_len);
 220        tmp_region = kzalloc(new_len + dma_ctx->dmatables_len, GFP_KERNEL);
 221        if (!tmp_region)
 222                return -ENOMEM;
 223
 224        /* Were there any previous tables that need to be preserved ? */
 225        if (*dmatables_region) {
 226                memcpy(tmp_region, *dmatables_region, dma_ctx->dmatables_len);
 227                kfree(*dmatables_region);
 228                *dmatables_region = NULL;
 229        }
 230
 231        *dmatables_region = tmp_region;
 232
 233        dma_ctx->dmatables_len += new_len;
 234
 235        return 0;
 236}
 237
 238/**
 239 *      sep_wait_transaction - Used for synchronizing transactions
 240 *      @sep: SEP device
 241 */
 242int sep_wait_transaction(struct sep_device *sep)
 243{
 244        int error = 0;
 245        DEFINE_WAIT(wait);
 246
 247        if (0 == test_and_set_bit(SEP_TRANSACTION_STARTED_LOCK_BIT,
 248                                &sep->in_use_flags)) {
 249                dev_dbg(&sep->pdev->dev,
 250                        "[PID%d] no transactions, returning\n",
 251                                current->pid);
 252                goto end_function_setpid;
 253        }
 254
 255        /*
 256         * Looping needed even for exclusive waitq entries
 257         * due to process wakeup latencies, previous process
 258         * might have already created another transaction.
 259         */
 260        for (;;) {
 261                /*
 262                 * Exclusive waitq entry, so that only one process is
 263                 * woken up from the queue at a time.
 264                 */
 265                prepare_to_wait_exclusive(&sep->event_transactions,
 266                                          &wait,
 267                                          TASK_INTERRUPTIBLE);
 268                if (0 == test_and_set_bit(SEP_TRANSACTION_STARTED_LOCK_BIT,
 269                                          &sep->in_use_flags)) {
 270                        dev_dbg(&sep->pdev->dev,
 271                                "[PID%d] no transactions, breaking\n",
 272                                        current->pid);
 273                        break;
 274                }
 275                dev_dbg(&sep->pdev->dev,
 276                        "[PID%d] transactions ongoing, sleeping\n",
 277                                current->pid);
 278                schedule();
 279                dev_dbg(&sep->pdev->dev, "[PID%d] woken up\n", current->pid);
 280
 281                if (signal_pending(current)) {
 282                        dev_dbg(&sep->pdev->dev, "[PID%d] received signal\n",
 283                                                        current->pid);
 284                        error = -EINTR;
 285                        goto end_function;
 286                }
 287        }
 288end_function_setpid:
 289        /*
 290         * The pid_doing_transaction indicates that this process
 291         * now owns the facilities to perform a transaction with
 292         * the SEP. While this process is performing a transaction,
 293         * no other process who has the SEP device open can perform
 294         * any transactions. This method allows more than one process
 295         * to have the device open at any given time, which provides
 296         * finer granularity for device utilization by multiple
 297         * processes.
 298         */
 299        /* Only one process is able to progress here at a time */
 300        sep->pid_doing_transaction = current->pid;
 301
 302end_function:
 303        finish_wait(&sep->event_transactions, &wait);
 304
 305        return error;
 306}
 307
 308/**
 309 * sep_check_transaction_owner - Checks if current process owns transaction
 310 * @sep: SEP device
 311 */
 312static inline int sep_check_transaction_owner(struct sep_device *sep)
 313{
 314        dev_dbg(&sep->pdev->dev, "[PID%d] transaction pid = %d\n",
 315                current->pid,
 316                sep->pid_doing_transaction);
 317
 318        if ((sep->pid_doing_transaction == 0) ||
 319                (current->pid != sep->pid_doing_transaction)) {
 320                return -EACCES;
 321        }
 322
 323        /* We own the transaction */
 324        return 0;
 325}
 326
 327#ifdef DEBUG
 328
 329/**
 330 * sep_dump_message - dump the message that is pending
 331 * @sep: SEP device
 332 * This will only print dump if DEBUG is set; it does
 333 * follow kernel debug print enabling
 334 */
 335static void _sep_dump_message(struct sep_device *sep)
 336{
 337        int count;
 338
 339        u32 *p = sep->shared_addr;
 340
 341        for (count = 0; count < 10 * 4; count += 4)
 342                dev_dbg(&sep->pdev->dev,
 343                        "[PID%d] Word %d of the message is %x\n",
 344                                current->pid, count/4, *p++);
 345}
 346
 347#endif
 348
 349/**
 350 * sep_map_and_alloc_shared_area -allocate shared block
 351 * @sep: security processor
 352 * @size: size of shared area
 353 */
 354static int sep_map_and_alloc_shared_area(struct sep_device *sep)
 355{
 356        sep->shared_addr = dma_alloc_coherent(&sep->pdev->dev,
 357                sep->shared_size,
 358                &sep->shared_bus, GFP_KERNEL);
 359
 360        if (!sep->shared_addr) {
 361                dev_dbg(&sep->pdev->dev,
 362                        "[PID%d] shared memory dma_alloc_coherent failed\n",
 363                                current->pid);
 364                return -ENOMEM;
 365        }
 366        dev_dbg(&sep->pdev->dev,
 367                "[PID%d] shared_addr %zx bytes @%p (bus %llx)\n",
 368                                current->pid,
 369                                sep->shared_size, sep->shared_addr,
 370                                (unsigned long long)sep->shared_bus);
 371        return 0;
 372}
 373
 374/**
 375 * sep_unmap_and_free_shared_area - free shared block
 376 * @sep: security processor
 377 */
 378static void sep_unmap_and_free_shared_area(struct sep_device *sep)
 379{
 380        dma_free_coherent(&sep->pdev->dev, sep->shared_size,
 381                                sep->shared_addr, sep->shared_bus);
 382}
 383
 384#ifdef DEBUG
 385
 386/**
 387 * sep_shared_bus_to_virt - convert bus/virt addresses
 388 * @sep: pointer to struct sep_device
 389 * @bus_address: address to convert
 390 *
 391 * Returns virtual address inside the shared area according
 392 * to the bus address.
 393 */
 394static void *sep_shared_bus_to_virt(struct sep_device *sep,
 395                                                dma_addr_t bus_address)
 396{
 397        return sep->shared_addr + (bus_address - sep->shared_bus);
 398}
 399
 400#endif
 401
 402/**
 403 * sep_open - device open method
 404 * @inode: inode of SEP device
 405 * @filp: file handle to SEP device
 406 *
 407 * Open method for the SEP device. Called when userspace opens
 408 * the SEP device node.
 409 *
 410 * Returns zero on success otherwise an error code.
 411 */
 412static int sep_open(struct inode *inode, struct file *filp)
 413{
 414        struct sep_device *sep;
 415        struct sep_private_data *priv;
 416
 417        dev_dbg(&sep_dev->pdev->dev, "[PID%d] open\n", current->pid);
 418
 419        if (filp->f_flags & O_NONBLOCK)
 420                return -ENOTSUPP;
 421
 422        /*
 423         * Get the SEP device structure and use it for the
 424         * private_data field in filp for other methods
 425         */
 426
 427        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 428        if (!priv)
 429                return -ENOMEM;
 430
 431        sep = sep_dev;
 432        priv->device = sep;
 433        filp->private_data = priv;
 434
 435        dev_dbg(&sep_dev->pdev->dev, "[PID%d] priv is 0x%p\n",
 436                                        current->pid, priv);
 437
 438        /* Anyone can open; locking takes place at transaction level */
 439        return 0;
 440}
 441
 442/**
 443 * sep_free_dma_table_data_handler - free DMA table
 444 * @sep: pointer to struct sep_device
 445 * @dma_ctx: dma context
 446 *
 447 * Handles the request to free DMA table for synchronic actions
 448 */
 449int sep_free_dma_table_data_handler(struct sep_device *sep,
 450                                           struct sep_dma_context **dma_ctx)
 451{
 452        int count;
 453        int dcb_counter;
 454        /* Pointer to the current dma_resource struct */
 455        struct sep_dma_resource *dma;
 456
 457        dev_dbg(&sep->pdev->dev,
 458                "[PID%d] sep_free_dma_table_data_handler\n",
 459                        current->pid);
 460
 461        if (!dma_ctx || !(*dma_ctx)) {
 462                /* No context or context already freed */
 463                dev_dbg(&sep->pdev->dev,
 464                        "[PID%d] no DMA context or context already freed\n",
 465                                current->pid);
 466
 467                return 0;
 468        }
 469
 470        dev_dbg(&sep->pdev->dev, "[PID%d] (*dma_ctx)->nr_dcb_creat 0x%x\n",
 471                                        current->pid,
 472                                        (*dma_ctx)->nr_dcb_creat);
 473
 474        for (dcb_counter = 0;
 475             dcb_counter < (*dma_ctx)->nr_dcb_creat; dcb_counter++) {
 476                dma = &(*dma_ctx)->dma_res_arr[dcb_counter];
 477
 478                /* Unmap and free input map array */
 479                if (dma->in_map_array) {
 480                        for (count = 0; count < dma->in_num_pages; count++) {
 481                                dma_unmap_page(&sep->pdev->dev,
 482                                        dma->in_map_array[count].dma_addr,
 483                                        dma->in_map_array[count].size,
 484                                        DMA_TO_DEVICE);
 485                        }
 486                        kfree(dma->in_map_array);
 487                }
 488
 489                /**
 490                 * Output is handled different. If
 491                 * this was a secure dma into restricted memory,
 492                 * then we skip this step altogether as restricted
 493                 * memory is not available to the o/s at all.
 494                 */
 495                if (!(*dma_ctx)->secure_dma && dma->out_map_array) {
 496
 497                        for (count = 0; count < dma->out_num_pages; count++) {
 498                                dma_unmap_page(&sep->pdev->dev,
 499                                        dma->out_map_array[count].dma_addr,
 500                                        dma->out_map_array[count].size,
 501                                        DMA_FROM_DEVICE);
 502                        }
 503                        kfree(dma->out_map_array);
 504                }
 505
 506                /* Free page cache for output */
 507                if (dma->in_page_array) {
 508                        for (count = 0; count < dma->in_num_pages; count++) {
 509                                flush_dcache_page(dma->in_page_array[count]);
 510                                page_cache_release(dma->in_page_array[count]);
 511                        }
 512                        kfree(dma->in_page_array);
 513                }
 514
 515                /* Again, we do this only for non secure dma */
 516                if (!(*dma_ctx)->secure_dma && dma->out_page_array) {
 517
 518                        for (count = 0; count < dma->out_num_pages; count++) {
 519                                if (!PageReserved(dma->out_page_array[count]))
 520
 521                                        SetPageDirty(dma->
 522                                        out_page_array[count]);
 523
 524                                flush_dcache_page(dma->out_page_array[count]);
 525                                page_cache_release(dma->out_page_array[count]);
 526                        }
 527                        kfree(dma->out_page_array);
 528                }
 529
 530                /**
 531                 * Note that here we use in_map_num_entries because we
 532                 * don't have a page array; the page array is generated
 533                 * only in the lock_user_pages, which is not called
 534                 * for kernel crypto, which is what the sg (scatter gather
 535                 * is used for exclusively)
 536                 */
 537                if (dma->src_sg) {
 538                        dma_unmap_sg(&sep->pdev->dev, dma->src_sg,
 539                                dma->in_map_num_entries, DMA_TO_DEVICE);
 540                        dma->src_sg = NULL;
 541                }
 542
 543                if (dma->dst_sg) {
 544                        dma_unmap_sg(&sep->pdev->dev, dma->dst_sg,
 545                                dma->in_map_num_entries, DMA_FROM_DEVICE);
 546                        dma->dst_sg = NULL;
 547                }
 548
 549                /* Reset all the values */
 550                dma->in_page_array = NULL;
 551                dma->out_page_array = NULL;
 552                dma->in_num_pages = 0;
 553                dma->out_num_pages = 0;
 554                dma->in_map_array = NULL;
 555                dma->out_map_array = NULL;
 556                dma->in_map_num_entries = 0;
 557                dma->out_map_num_entries = 0;
 558        }
 559
 560        (*dma_ctx)->nr_dcb_creat = 0;
 561        (*dma_ctx)->num_lli_tables_created = 0;
 562
 563        kfree(*dma_ctx);
 564        *dma_ctx = NULL;
 565
 566        dev_dbg(&sep->pdev->dev,
 567                "[PID%d] sep_free_dma_table_data_handler end\n",
 568                        current->pid);
 569
 570        return 0;
 571}
 572
 573/**
 574 * sep_end_transaction_handler - end transaction
 575 * @sep: pointer to struct sep_device
 576 * @dma_ctx: DMA context
 577 * @call_status: Call status
 578 *
 579 * This API handles the end transaction request.
 580 */
 581static int sep_end_transaction_handler(struct sep_device *sep,
 582                                       struct sep_dma_context **dma_ctx,
 583                                       struct sep_call_status *call_status,
 584                                       struct sep_queue_info **my_queue_elem)
 585{
 586        dev_dbg(&sep->pdev->dev, "[PID%d] ending transaction\n", current->pid);
 587
 588        /*
 589         * Extraneous transaction clearing would mess up PM
 590         * device usage counters and SEP would get suspended
 591         * just before we send a command to SEP in the next
 592         * transaction
 593         * */
 594        if (sep_check_transaction_owner(sep)) {
 595                dev_dbg(&sep->pdev->dev, "[PID%d] not transaction owner\n",
 596                                                current->pid);
 597                return 0;
 598        }
 599
 600        /* Update queue status */
 601        sep_queue_status_remove(sep, my_queue_elem);
 602
 603        /* Check that all the DMA resources were freed */
 604        if (dma_ctx)
 605                sep_free_dma_table_data_handler(sep, dma_ctx);
 606
 607        /* Reset call status for next transaction */
 608        if (call_status)
 609                call_status->status = 0;
 610
 611        /* Clear the message area to avoid next transaction reading
 612         * sensitive results from previous transaction */
 613        memset(sep->shared_addr, 0,
 614               SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES);
 615
 616        /* start suspend delay */
 617#ifdef SEP_ENABLE_RUNTIME_PM
 618        if (sep->in_use) {
 619                sep->in_use = 0;
 620                pm_runtime_mark_last_busy(&sep->pdev->dev);
 621                pm_runtime_put_autosuspend(&sep->pdev->dev);
 622        }
 623#endif
 624
 625        clear_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags);
 626        sep->pid_doing_transaction = 0;
 627
 628        /* Now it's safe for next process to proceed */
 629        dev_dbg(&sep->pdev->dev, "[PID%d] waking up next transaction\n",
 630                                        current->pid);
 631        clear_bit(SEP_TRANSACTION_STARTED_LOCK_BIT, &sep->in_use_flags);
 632        wake_up(&sep->event_transactions);
 633
 634        return 0;
 635}
 636
 637
 638/**
 639 * sep_release - close a SEP device
 640 * @inode: inode of SEP device
 641 * @filp: file handle being closed
 642 *
 643 * Called on the final close of a SEP device.
 644 */
 645static int sep_release(struct inode *inode, struct file *filp)
 646{
 647        struct sep_private_data * const private_data = filp->private_data;
 648        struct sep_call_status *call_status = &private_data->call_status;
 649        struct sep_device *sep = private_data->device;
 650        struct sep_dma_context **dma_ctx = &private_data->dma_ctx;
 651        struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem;
 652
 653        dev_dbg(&sep->pdev->dev, "[PID%d] release\n", current->pid);
 654
 655        sep_end_transaction_handler(sep, dma_ctx, call_status,
 656                my_queue_elem);
 657
 658        kfree(filp->private_data);
 659
 660        return 0;
 661}
 662
 663/**
 664 * sep_mmap -  maps the shared area to user space
 665 * @filp: pointer to struct file
 666 * @vma: pointer to vm_area_struct
 667 *
 668 * Called on an mmap of our space via the normal SEP device
 669 */
 670static int sep_mmap(struct file *filp, struct vm_area_struct *vma)
 671{
 672        struct sep_private_data * const private_data = filp->private_data;
 673        struct sep_call_status *call_status = &private_data->call_status;
 674        struct sep_device *sep = private_data->device;
 675        struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem;
 676        dma_addr_t bus_addr;
 677        unsigned long error = 0;
 678
 679        dev_dbg(&sep->pdev->dev, "[PID%d] sep_mmap\n", current->pid);
 680
 681        /* Set the transaction busy (own the device) */
 682        /*
 683         * Problem for multithreaded applications is that here we're
 684         * possibly going to sleep while holding a write lock on
 685         * current->mm->mmap_sem, which will cause deadlock for ongoing
 686         * transaction trying to create DMA tables
 687         */
 688        error = sep_wait_transaction(sep);
 689        if (error)
 690                /* Interrupted by signal, don't clear transaction */
 691                goto end_function;
 692
 693        /* Clear the message area to avoid next transaction reading
 694         * sensitive results from previous transaction */
 695        memset(sep->shared_addr, 0,
 696               SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES);
 697
 698        /*
 699         * Check that the size of the mapped range is as the size of the message
 700         * shared area
 701         */
 702        if ((vma->vm_end - vma->vm_start) > SEP_DRIVER_MMMAP_AREA_SIZE) {
 703                error = -EINVAL;
 704                goto end_function_with_error;
 705        }
 706
 707        dev_dbg(&sep->pdev->dev, "[PID%d] shared_addr is %p\n",
 708                                        current->pid, sep->shared_addr);
 709
 710        /* Get bus address */
 711        bus_addr = sep->shared_bus;
 712
 713        if (remap_pfn_range(vma, vma->vm_start, bus_addr >> PAGE_SHIFT,
 714                vma->vm_end - vma->vm_start, vma->vm_page_prot)) {
 715                dev_dbg(&sep->pdev->dev, "[PID%d] remap_pfn_range failed\n",
 716                                                current->pid);
 717                error = -EAGAIN;
 718                goto end_function_with_error;
 719        }
 720
 721        /* Update call status */
 722        set_bit(SEP_LEGACY_MMAP_DONE_OFFSET, &call_status->status);
 723
 724        goto end_function;
 725
 726end_function_with_error:
 727        /* Clear our transaction */
 728        sep_end_transaction_handler(sep, NULL, call_status,
 729                my_queue_elem);
 730
 731end_function:
 732        return error;
 733}
 734
 735/**
 736 * sep_poll - poll handler
 737 * @filp:       pointer to struct file
 738 * @wait:       pointer to poll_table
 739 *
 740 * Called by the OS when the kernel is asked to do a poll on
 741 * a SEP file handle.
 742 */
 743static unsigned int sep_poll(struct file *filp, poll_table *wait)
 744{
 745        struct sep_private_data * const private_data = filp->private_data;
 746        struct sep_call_status *call_status = &private_data->call_status;
 747        struct sep_device *sep = private_data->device;
 748        u32 mask = 0;
 749        u32 retval = 0;
 750        u32 retval2 = 0;
 751        unsigned long lock_irq_flag;
 752
 753        /* Am I the process that owns the transaction? */
 754        if (sep_check_transaction_owner(sep)) {
 755                dev_dbg(&sep->pdev->dev, "[PID%d] poll pid not owner\n",
 756                                                current->pid);
 757                mask = POLLERR;
 758                goto end_function;
 759        }
 760
 761        /* Check if send command or send_reply were activated previously */
 762        if (0 == test_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET,
 763                          &call_status->status)) {
 764                dev_warn(&sep->pdev->dev, "[PID%d] sendmsg not called\n",
 765                                                current->pid);
 766                mask = POLLERR;
 767                goto end_function;
 768        }
 769
 770
 771        /* Add the event to the polling wait table */
 772        dev_dbg(&sep->pdev->dev, "[PID%d] poll: calling wait sep_event\n",
 773                                        current->pid);
 774
 775        poll_wait(filp, &sep->event_interrupt, wait);
 776
 777        dev_dbg(&sep->pdev->dev,
 778                "[PID%d] poll: send_ct is %lx reply ct is %lx\n",
 779                        current->pid, sep->send_ct, sep->reply_ct);
 780
 781        /* Check if error occurred during poll */
 782        retval2 = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR3_REG_ADDR);
 783        if ((retval2 != 0x0) && (retval2 != 0x8)) {
 784                dev_dbg(&sep->pdev->dev, "[PID%d] poll; poll error %x\n",
 785                                                current->pid, retval2);
 786                mask |= POLLERR;
 787                goto end_function;
 788        }
 789
 790        spin_lock_irqsave(&sep->snd_rply_lck, lock_irq_flag);
 791
 792        if (sep->send_ct == sep->reply_ct) {
 793                spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag);
 794                retval = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR);
 795                dev_dbg(&sep->pdev->dev,
 796                        "[PID%d] poll: data ready check (GPR2)  %x\n",
 797                                current->pid, retval);
 798
 799                /* Check if printf request  */
 800                if ((retval >> 30) & 0x1) {
 801                        dev_dbg(&sep->pdev->dev,
 802                                "[PID%d] poll: SEP printf request\n",
 803                                        current->pid);
 804                        goto end_function;
 805                }
 806
 807                /* Check if the this is SEP reply or request */
 808                if (retval >> 31) {
 809                        dev_dbg(&sep->pdev->dev,
 810                                "[PID%d] poll: SEP request\n",
 811                                        current->pid);
 812                } else {
 813                        dev_dbg(&sep->pdev->dev,
 814                                "[PID%d] poll: normal return\n",
 815                                        current->pid);
 816                        sep_dump_message(sep);
 817                        dev_dbg(&sep->pdev->dev,
 818                                "[PID%d] poll; SEP reply POLLIN|POLLRDNORM\n",
 819                                        current->pid);
 820                        mask |= POLLIN | POLLRDNORM;
 821                }
 822                set_bit(SEP_LEGACY_POLL_DONE_OFFSET, &call_status->status);
 823        } else {
 824                spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag);
 825                dev_dbg(&sep->pdev->dev,
 826                        "[PID%d] poll; no reply; returning mask of 0\n",
 827                                current->pid);
 828                mask = 0;
 829        }
 830
 831end_function:
 832        return mask;
 833}
 834
 835/**
 836 * sep_time_address - address in SEP memory of time
 837 * @sep: SEP device we want the address from
 838 *
 839 * Return the address of the two dwords in memory used for time
 840 * setting.
 841 */
 842static u32 *sep_time_address(struct sep_device *sep)
 843{
 844        return sep->shared_addr +
 845                SEP_DRIVER_SYSTEM_TIME_MEMORY_OFFSET_IN_BYTES;
 846}
 847
 848/**
 849 * sep_set_time - set the SEP time
 850 * @sep: the SEP we are setting the time for
 851 *
 852 * Calculates time and sets it at the predefined address.
 853 * Called with the SEP mutex held.
 854 */
 855static unsigned long sep_set_time(struct sep_device *sep)
 856{
 857        struct timeval time;
 858        u32 *time_addr; /* Address of time as seen by the kernel */
 859
 860
 861        do_gettimeofday(&time);
 862
 863        /* Set value in the SYSTEM MEMORY offset */
 864        time_addr = sep_time_address(sep);
 865
 866        time_addr[0] = SEP_TIME_VAL_TOKEN;
 867        time_addr[1] = time.tv_sec;
 868
 869        dev_dbg(&sep->pdev->dev, "[PID%d] time.tv_sec is %lu\n",
 870                                        current->pid, time.tv_sec);
 871        dev_dbg(&sep->pdev->dev, "[PID%d] time_addr is %p\n",
 872                                        current->pid, time_addr);
 873        dev_dbg(&sep->pdev->dev, "[PID%d] sep->shared_addr is %p\n",
 874                                        current->pid, sep->shared_addr);
 875
 876        return time.tv_sec;
 877}
 878
 879/**
 880 * sep_send_command_handler - kick off a command
 881 * @sep: SEP being signalled
 882 *
 883 * This function raises interrupt to SEP that signals that is has a new
 884 * command from the host
 885 *
 886 * Note that this function does fall under the ioctl lock
 887 */
 888int sep_send_command_handler(struct sep_device *sep)
 889{
 890        unsigned long lock_irq_flag;
 891        u32 *msg_pool;
 892        int error = 0;
 893
 894        /* Basic sanity check; set msg pool to start of shared area */
 895        msg_pool = (u32 *)sep->shared_addr;
 896        msg_pool += 2;
 897
 898        /* Look for start msg token */
 899        if (*msg_pool != SEP_START_MSG_TOKEN) {
 900                dev_warn(&sep->pdev->dev, "start message token not present\n");
 901                error = -EPROTO;
 902                goto end_function;
 903        }
 904
 905        /* Do we have a reasonable size? */
 906        msg_pool += 1;
 907        if ((*msg_pool < 2) ||
 908                (*msg_pool > SEP_DRIVER_MAX_MESSAGE_SIZE_IN_BYTES)) {
 909
 910                dev_warn(&sep->pdev->dev, "invalid message size\n");
 911                error = -EPROTO;
 912                goto end_function;
 913        }
 914
 915        /* Does the command look reasonable? */
 916        msg_pool += 1;
 917        if (*msg_pool < 2) {
 918                dev_warn(&sep->pdev->dev, "invalid message opcode\n");
 919                error = -EPROTO;
 920                goto end_function;
 921        }
 922
 923#if defined(CONFIG_PM_RUNTIME) && defined(SEP_ENABLE_RUNTIME_PM)
 924        dev_dbg(&sep->pdev->dev, "[PID%d] before pm sync status 0x%X\n",
 925                                        current->pid,
 926                                        sep->pdev->dev.power.runtime_status);
 927        sep->in_use = 1; /* device is about to be used */
 928        pm_runtime_get_sync(&sep->pdev->dev);
 929#endif
 930
 931        if (test_and_set_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags)) {
 932                error = -EPROTO;
 933                goto end_function;
 934        }
 935        sep->in_use = 1; /* device is about to be used */
 936        sep_set_time(sep);
 937
 938        sep_dump_message(sep);
 939
 940        /* Update counter */
 941        spin_lock_irqsave(&sep->snd_rply_lck, lock_irq_flag);
 942        sep->send_ct++;
 943        spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag);
 944
 945        dev_dbg(&sep->pdev->dev,
 946                "[PID%d] sep_send_command_handler send_ct %lx reply_ct %lx\n",
 947                        current->pid, sep->send_ct, sep->reply_ct);
 948
 949        /* Send interrupt to SEP */
 950        sep_write_reg(sep, HW_HOST_HOST_SEP_GPR0_REG_ADDR, 0x2);
 951
 952end_function:
 953        return error;
 954}
 955
 956/**
 957 *      sep_crypto_dma -
 958 *      @sep: pointer to struct sep_device
 959 *      @sg: pointer to struct scatterlist
 960 *      @direction:
 961 *      @dma_maps: pointer to place a pointer to array of dma maps
 962 *       This is filled in; anything previous there will be lost
 963 *       The structure for dma maps is sep_dma_map
 964 *      @returns number of dma maps on success; negative on error
 965 *
 966 *      This creates the dma table from the scatterlist
 967 *      It is used only for kernel crypto as it works with scatterlists
 968 *      representation of data buffers
 969 *
 970 */
 971static int sep_crypto_dma(
 972        struct sep_device *sep,
 973        struct scatterlist *sg,
 974        struct sep_dma_map **dma_maps,
 975        enum dma_data_direction direction)
 976{
 977        struct scatterlist *temp_sg;
 978
 979        u32 count_segment;
 980        u32 count_mapped;
 981        struct sep_dma_map *sep_dma;
 982        int ct1;
 983
 984        if (sg->length == 0)
 985                return 0;
 986
 987        /* Count the segments */
 988        temp_sg = sg;
 989        count_segment = 0;
 990        while (temp_sg) {
 991                count_segment += 1;
 992                temp_sg = scatterwalk_sg_next(temp_sg);
 993        }
 994        dev_dbg(&sep->pdev->dev,
 995                "There are (hex) %x segments in sg\n", count_segment);
 996
 997        /* DMA map segments */
 998        count_mapped = dma_map_sg(&sep->pdev->dev, sg,
 999                count_segment, direction);
1000
1001        dev_dbg(&sep->pdev->dev,
1002                "There are (hex) %x maps in sg\n", count_mapped);
1003
1004        if (count_mapped == 0) {
1005                dev_dbg(&sep->pdev->dev, "Cannot dma_map_sg\n");
1006                return -ENOMEM;
1007        }
1008
1009        sep_dma = kmalloc(sizeof(struct sep_dma_map) *
1010                count_mapped, GFP_ATOMIC);
1011
1012        if (sep_dma == NULL) {
1013                dev_dbg(&sep->pdev->dev, "Cannot allocate dma_maps\n");
1014                return -ENOMEM;
1015        }
1016
1017        for_each_sg(sg, temp_sg, count_mapped, ct1) {
1018                sep_dma[ct1].dma_addr = sg_dma_address(temp_sg);
1019                sep_dma[ct1].size = sg_dma_len(temp_sg);
1020                dev_dbg(&sep->pdev->dev, "(all hex) map %x dma %lx len %lx\n",
1021                        ct1, (unsigned long)sep_dma[ct1].dma_addr,
1022                        (unsigned long)sep_dma[ct1].size);
1023                }
1024
1025        *dma_maps = sep_dma;
1026        return count_mapped;
1027
1028}
1029
1030/**
1031 *      sep_crypto_lli -
1032 *      @sep: pointer to struct sep_device
1033 *      @sg: pointer to struct scatterlist
1034 *      @data_size: total data size
1035 *      @direction:
1036 *      @dma_maps: pointer to place a pointer to array of dma maps
1037 *       This is filled in; anything previous there will be lost
1038 *       The structure for dma maps is sep_dma_map
1039 *      @lli_maps: pointer to place a pointer to array of lli maps
1040 *       This is filled in; anything previous there will be lost
1041 *       The structure for dma maps is sep_dma_map
1042 *      @returns number of dma maps on success; negative on error
1043 *
1044 *      This creates the LLI table from the scatterlist
1045 *      It is only used for kernel crypto as it works exclusively
1046 *      with scatterlists (struct scatterlist) representation of
1047 *      data buffers
1048 */
1049static int sep_crypto_lli(
1050        struct sep_device *sep,
1051        struct scatterlist *sg,
1052        struct sep_dma_map **maps,
1053        struct sep_lli_entry **llis,
1054        u32 data_size,
1055        enum dma_data_direction direction)
1056{
1057
1058        int ct1;
1059        struct sep_lli_entry *sep_lli;
1060        struct sep_dma_map *sep_map;
1061
1062        int nbr_ents;
1063
1064        nbr_ents = sep_crypto_dma(sep, sg, maps, direction);
1065        if (nbr_ents <= 0) {
1066                dev_dbg(&sep->pdev->dev, "crypto_dma failed %x\n",
1067                        nbr_ents);
1068                return nbr_ents;
1069        }
1070
1071        sep_map = *maps;
1072
1073        sep_lli = kmalloc(sizeof(struct sep_lli_entry) * nbr_ents, GFP_ATOMIC);
1074
1075        if (sep_lli == NULL) {
1076                dev_dbg(&sep->pdev->dev, "Cannot allocate lli_maps\n");
1077
1078                kfree(*maps);
1079                *maps = NULL;
1080                return -ENOMEM;
1081        }
1082
1083        for (ct1 = 0; ct1 < nbr_ents; ct1 += 1) {
1084                sep_lli[ct1].bus_address = (u32)sep_map[ct1].dma_addr;
1085
1086                /* Maximum for page is total data size */
1087                if (sep_map[ct1].size > data_size)
1088                        sep_map[ct1].size = data_size;
1089
1090                sep_lli[ct1].block_size = (u32)sep_map[ct1].size;
1091        }
1092
1093        *llis = sep_lli;
1094        return nbr_ents;
1095}
1096
1097/**
1098 *      sep_lock_kernel_pages - map kernel pages for DMA
1099 *      @sep: pointer to struct sep_device
1100 *      @kernel_virt_addr: address of data buffer in kernel
1101 *      @data_size: size of data
1102 *      @lli_array_ptr: lli array
1103 *      @in_out_flag: input into device or output from device
1104 *
1105 *      This function locks all the physical pages of the kernel virtual buffer
1106 *      and construct a basic lli  array, where each entry holds the physical
1107 *      page address and the size that application data holds in this page
1108 *      This function is used only during kernel crypto mod calls from within
1109 *      the kernel (when ioctl is not used)
1110 *
1111 *      This is used only for kernel crypto. Kernel pages
1112 *      are handled differently as they are done via
1113 *      scatter gather lists (struct scatterlist)
1114 */
1115static int sep_lock_kernel_pages(struct sep_device *sep,
1116        unsigned long kernel_virt_addr,
1117        u32 data_size,
1118        struct sep_lli_entry **lli_array_ptr,
1119        int in_out_flag,
1120        struct sep_dma_context *dma_ctx)
1121
1122{
1123        u32 num_pages;
1124        struct scatterlist *sg;
1125
1126        /* Array of lli */
1127        struct sep_lli_entry *lli_array;
1128        /* Map array */
1129        struct sep_dma_map *map_array;
1130
1131        enum dma_data_direction direction;
1132
1133        lli_array = NULL;
1134        map_array = NULL;
1135
1136        if (in_out_flag == SEP_DRIVER_IN_FLAG) {
1137                direction = DMA_TO_DEVICE;
1138                sg = dma_ctx->src_sg;
1139        } else {
1140                direction = DMA_FROM_DEVICE;
1141                sg = dma_ctx->dst_sg;
1142        }
1143
1144        num_pages = sep_crypto_lli(sep, sg, &map_array, &lli_array,
1145                data_size, direction);
1146
1147        if (num_pages <= 0) {
1148                dev_dbg(&sep->pdev->dev, "sep_crypto_lli returned error %x\n",
1149                        num_pages);
1150                return -ENOMEM;
1151        }
1152
1153        /* Put mapped kernel sg into kernel resource array */
1154
1155        /* Set output params according to the in_out flag */
1156        if (in_out_flag == SEP_DRIVER_IN_FLAG) {
1157                *lli_array_ptr = lli_array;
1158                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages =
1159                                                                num_pages;
1160                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array =
1161                                                                NULL;
1162                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array =
1163                                                                map_array;
1164                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_num_entries =
1165                                                                num_pages;
1166                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].src_sg =
1167                        dma_ctx->src_sg;
1168        } else {
1169                *lli_array_ptr = lli_array;
1170                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages =
1171                                                                num_pages;
1172                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array =
1173                                                                NULL;
1174                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array =
1175                                                                map_array;
1176                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].
1177                                        out_map_num_entries = num_pages;
1178                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].dst_sg =
1179                        dma_ctx->dst_sg;
1180        }
1181
1182        return 0;
1183}
1184
1185/**
1186 * sep_lock_user_pages - lock and map user pages for DMA
1187 * @sep: pointer to struct sep_device
1188 * @app_virt_addr: user memory data buffer
1189 * @data_size: size of data buffer
1190 * @lli_array_ptr: lli array
1191 * @in_out_flag: input or output to device
1192 *
1193 * This function locks all the physical pages of the application
1194 * virtual buffer and construct a basic lli  array, where each entry
1195 * holds the physical page address and the size that application
1196 * data holds in this physical pages
1197 */
1198static int sep_lock_user_pages(struct sep_device *sep,
1199        u32 app_virt_addr,
1200        u32 data_size,
1201        struct sep_lli_entry **lli_array_ptr,
1202        int in_out_flag,
1203        struct sep_dma_context *dma_ctx)
1204
1205{
1206        int error = 0;
1207        u32 count;
1208        int result;
1209        /* The the page of the end address of the user space buffer */
1210        u32 end_page;
1211        /* The page of the start address of the user space buffer */
1212        u32 start_page;
1213        /* The range in pages */
1214        u32 num_pages;
1215        /* Array of pointers to page */
1216        struct page **page_array;
1217        /* Array of lli */
1218        struct sep_lli_entry *lli_array;
1219        /* Map array */
1220        struct sep_dma_map *map_array;
1221
1222        /* Set start and end pages and num pages */
1223        end_page = (app_virt_addr + data_size - 1) >> PAGE_SHIFT;
1224        start_page = app_virt_addr >> PAGE_SHIFT;
1225        num_pages = end_page - start_page + 1;
1226
1227        dev_dbg(&sep->pdev->dev,
1228                "[PID%d] lock user pages app_virt_addr is %x\n",
1229                        current->pid, app_virt_addr);
1230
1231        dev_dbg(&sep->pdev->dev, "[PID%d] data_size is (hex) %x\n",
1232                                        current->pid, data_size);
1233        dev_dbg(&sep->pdev->dev, "[PID%d] start_page is (hex) %x\n",
1234                                        current->pid, start_page);
1235        dev_dbg(&sep->pdev->dev, "[PID%d] end_page is (hex) %x\n",
1236                                        current->pid, end_page);
1237        dev_dbg(&sep->pdev->dev, "[PID%d] num_pages is (hex) %x\n",
1238                                        current->pid, num_pages);
1239
1240        /* Allocate array of pages structure pointers */
1241        page_array = kmalloc_array(num_pages, sizeof(struct page *),
1242                                   GFP_ATOMIC);
1243        if (!page_array) {
1244                error = -ENOMEM;
1245                goto end_function;
1246        }
1247
1248        map_array = kmalloc_array(num_pages, sizeof(struct sep_dma_map),
1249                                  GFP_ATOMIC);
1250        if (!map_array) {
1251                error = -ENOMEM;
1252                goto end_function_with_error1;
1253        }
1254
1255        lli_array = kmalloc_array(num_pages, sizeof(struct sep_lli_entry),
1256                                  GFP_ATOMIC);
1257        if (!lli_array) {
1258                error = -ENOMEM;
1259                goto end_function_with_error2;
1260        }
1261
1262        /* Convert the application virtual address into a set of physical */
1263        result = get_user_pages_fast(app_virt_addr, num_pages,
1264                ((in_out_flag == SEP_DRIVER_IN_FLAG) ? 0 : 1), page_array);
1265
1266        /* Check the number of pages locked - if not all then exit with error */
1267        if (result != num_pages) {
1268                dev_warn(&sep->pdev->dev,
1269                        "[PID%d] not all pages locked by get_user_pages, "
1270                        "result 0x%X, num_pages 0x%X\n",
1271                                current->pid, result, num_pages);
1272                error = -ENOMEM;
1273                goto end_function_with_error3;
1274        }
1275
1276        dev_dbg(&sep->pdev->dev, "[PID%d] get_user_pages succeeded\n",
1277                                        current->pid);
1278
1279        /*
1280         * Fill the array using page array data and
1281         * map the pages - this action will also flush the cache as needed
1282         */
1283        for (count = 0; count < num_pages; count++) {
1284                /* Fill the map array */
1285                map_array[count].dma_addr =
1286                        dma_map_page(&sep->pdev->dev, page_array[count],
1287                        0, PAGE_SIZE, DMA_BIDIRECTIONAL);
1288
1289                map_array[count].size = PAGE_SIZE;
1290
1291                /* Fill the lli array entry */
1292                lli_array[count].bus_address = (u32)map_array[count].dma_addr;
1293                lli_array[count].block_size = PAGE_SIZE;
1294
1295                dev_dbg(&sep->pdev->dev,
1296                        "[PID%d] lli_array[%x].bus_address is %08lx, "
1297                        "lli_array[%x].block_size is (hex) %x\n", current->pid,
1298                        count, (unsigned long)lli_array[count].bus_address,
1299                        count, lli_array[count].block_size);
1300        }
1301
1302        /* Check the offset for the first page */
1303        lli_array[0].bus_address =
1304                lli_array[0].bus_address + (app_virt_addr & (~PAGE_MASK));
1305
1306        /* Check that not all the data is in the first page only */
1307        if ((PAGE_SIZE - (app_virt_addr & (~PAGE_MASK))) >= data_size)
1308                lli_array[0].block_size = data_size;
1309        else
1310                lli_array[0].block_size =
1311                        PAGE_SIZE - (app_virt_addr & (~PAGE_MASK));
1312
1313                dev_dbg(&sep->pdev->dev,
1314                        "[PID%d] After check if page 0 has all data\n",
1315                        current->pid);
1316                dev_dbg(&sep->pdev->dev,
1317                        "[PID%d] lli_array[0].bus_address is (hex) %08lx, "
1318                        "lli_array[0].block_size is (hex) %x\n",
1319                        current->pid,
1320                        (unsigned long)lli_array[0].bus_address,
1321                        lli_array[0].block_size);
1322
1323
1324        /* Check the size of the last page */
1325        if (num_pages > 1) {
1326                lli_array[num_pages - 1].block_size =
1327                        (app_virt_addr + data_size) & (~PAGE_MASK);
1328                if (lli_array[num_pages - 1].block_size == 0)
1329                        lli_array[num_pages - 1].block_size = PAGE_SIZE;
1330
1331                dev_dbg(&sep->pdev->dev,
1332                        "[PID%d] After last page size adjustment\n",
1333                        current->pid);
1334                dev_dbg(&sep->pdev->dev,
1335                        "[PID%d] lli_array[%x].bus_address is (hex) %08lx, "
1336                        "lli_array[%x].block_size is (hex) %x\n",
1337                        current->pid,
1338                        num_pages - 1,
1339                        (unsigned long)lli_array[num_pages - 1].bus_address,
1340                        num_pages - 1,
1341                        lli_array[num_pages - 1].block_size);
1342        }
1343
1344        /* Set output params according to the in_out flag */
1345        if (in_out_flag == SEP_DRIVER_IN_FLAG) {
1346                *lli_array_ptr = lli_array;
1347                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages =
1348                                                                num_pages;
1349                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array =
1350                                                                page_array;
1351                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array =
1352                                                                map_array;
1353                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_num_entries =
1354                                                                num_pages;
1355                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].src_sg = NULL;
1356        } else {
1357                *lli_array_ptr = lli_array;
1358                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages =
1359                                                                num_pages;
1360                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array =
1361                                                                page_array;
1362                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array =
1363                                                                map_array;
1364                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].
1365                                        out_map_num_entries = num_pages;
1366                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].dst_sg = NULL;
1367        }
1368        goto end_function;
1369
1370end_function_with_error3:
1371        /* Free lli array */
1372        kfree(lli_array);
1373
1374end_function_with_error2:
1375        kfree(map_array);
1376
1377end_function_with_error1:
1378        /* Free page array */
1379        kfree(page_array);
1380
1381end_function:
1382        return error;
1383}
1384
1385/**
1386 *      sep_lli_table_secure_dma - get lli array for IMR addresses
1387 *      @sep: pointer to struct sep_device
1388 *      @app_virt_addr: user memory data buffer
1389 *      @data_size: size of data buffer
1390 *      @lli_array_ptr: lli array
1391 *      @in_out_flag: not used
1392 *      @dma_ctx: pointer to struct sep_dma_context
1393 *
1394 *      This function creates lli tables for outputting data to
1395 *      IMR memory, which is memory that cannot be accessed by the
1396 *      the x86 processor.
1397 */
1398static int sep_lli_table_secure_dma(struct sep_device *sep,
1399        u32 app_virt_addr,
1400        u32 data_size,
1401        struct sep_lli_entry **lli_array_ptr,
1402        int in_out_flag,
1403        struct sep_dma_context *dma_ctx)
1404
1405{
1406        int error = 0;
1407        u32 count;
1408        /* The the page of the end address of the user space buffer */
1409        u32 end_page;
1410        /* The page of the start address of the user space buffer */
1411        u32 start_page;
1412        /* The range in pages */
1413        u32 num_pages;
1414        /* Array of lli */
1415        struct sep_lli_entry *lli_array;
1416
1417        /* Set start and end pages and num pages */
1418        end_page = (app_virt_addr + data_size - 1) >> PAGE_SHIFT;
1419        start_page = app_virt_addr >> PAGE_SHIFT;
1420        num_pages = end_page - start_page + 1;
1421
1422        dev_dbg(&sep->pdev->dev,
1423                "[PID%d] lock user pages  app_virt_addr is %x\n",
1424                current->pid, app_virt_addr);
1425
1426        dev_dbg(&sep->pdev->dev, "[PID%d] data_size is (hex) %x\n",
1427                current->pid, data_size);
1428        dev_dbg(&sep->pdev->dev, "[PID%d] start_page is (hex) %x\n",
1429                current->pid, start_page);
1430        dev_dbg(&sep->pdev->dev, "[PID%d] end_page is (hex) %x\n",
1431                current->pid, end_page);
1432        dev_dbg(&sep->pdev->dev, "[PID%d] num_pages is (hex) %x\n",
1433                current->pid, num_pages);
1434
1435        lli_array = kmalloc_array(num_pages, sizeof(struct sep_lli_entry),
1436                                  GFP_ATOMIC);
1437        if (!lli_array)
1438                return -ENOMEM;
1439
1440        /*
1441         * Fill the lli_array
1442         */
1443        start_page = start_page << PAGE_SHIFT;
1444        for (count = 0; count < num_pages; count++) {
1445                /* Fill the lli array entry */
1446                lli_array[count].bus_address = start_page;
1447                lli_array[count].block_size = PAGE_SIZE;
1448
1449                start_page += PAGE_SIZE;
1450
1451                dev_dbg(&sep->pdev->dev,
1452                        "[PID%d] lli_array[%x].bus_address is %08lx, "
1453                        "lli_array[%x].block_size is (hex) %x\n",
1454                        current->pid,
1455                        count, (unsigned long)lli_array[count].bus_address,
1456                        count, lli_array[count].block_size);
1457        }
1458
1459        /* Check the offset for the first page */
1460        lli_array[0].bus_address =
1461                lli_array[0].bus_address + (app_virt_addr & (~PAGE_MASK));
1462
1463        /* Check that not all the data is in the first page only */
1464        if ((PAGE_SIZE - (app_virt_addr & (~PAGE_MASK))) >= data_size)
1465                lli_array[0].block_size = data_size;
1466        else
1467                lli_array[0].block_size =
1468                        PAGE_SIZE - (app_virt_addr & (~PAGE_MASK));
1469
1470        dev_dbg(&sep->pdev->dev,
1471                "[PID%d] After check if page 0 has all data\n"
1472                "lli_array[0].bus_address is (hex) %08lx, "
1473                "lli_array[0].block_size is (hex) %x\n",
1474                current->pid,
1475                (unsigned long)lli_array[0].bus_address,
1476                lli_array[0].block_size);
1477
1478        /* Check the size of the last page */
1479        if (num_pages > 1) {
1480                lli_array[num_pages - 1].block_size =
1481                        (app_virt_addr + data_size) & (~PAGE_MASK);
1482                if (lli_array[num_pages - 1].block_size == 0)
1483                        lli_array[num_pages - 1].block_size = PAGE_SIZE;
1484
1485                dev_dbg(&sep->pdev->dev,
1486                        "[PID%d] After last page size adjustment\n"
1487                        "lli_array[%x].bus_address is (hex) %08lx, "
1488                        "lli_array[%x].block_size is (hex) %x\n",
1489                        current->pid, num_pages - 1,
1490                        (unsigned long)lli_array[num_pages - 1].bus_address,
1491                        num_pages - 1,
1492                        lli_array[num_pages - 1].block_size);
1493        }
1494        *lli_array_ptr = lli_array;
1495        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages = num_pages;
1496        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = NULL;
1497        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array = NULL;
1498        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_num_entries = 0;
1499
1500        return error;
1501}
1502
1503/**
1504 * sep_calculate_lli_table_max_size - size the LLI table
1505 * @sep: pointer to struct sep_device
1506 * @lli_in_array_ptr
1507 * @num_array_entries
1508 * @last_table_flag
1509 *
1510 * This function calculates the size of data that can be inserted into
1511 * the lli table from this array, such that either the table is full
1512 * (all entries are entered), or there are no more entries in the
1513 * lli array
1514 */
1515static u32 sep_calculate_lli_table_max_size(struct sep_device *sep,
1516        struct sep_lli_entry *lli_in_array_ptr,
1517        u32 num_array_entries,
1518        u32 *last_table_flag)
1519{
1520        u32 counter;
1521        /* Table data size */
1522        u32 table_data_size = 0;
1523        /* Data size for the next table */
1524        u32 next_table_data_size;
1525
1526        *last_table_flag = 0;
1527
1528        /*
1529         * Calculate the data in the out lli table till we fill the whole
1530         * table or till the data has ended
1531         */
1532        for (counter = 0;
1533                (counter < (SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP - 1)) &&
1534                        (counter < num_array_entries); counter++)
1535                table_data_size += lli_in_array_ptr[counter].block_size;
1536
1537        /*
1538         * Check if we reached the last entry,
1539         * meaning this ia the last table to build,
1540         * and no need to check the block alignment
1541         */
1542        if (counter == num_array_entries) {
1543                /* Set the last table flag */
1544                *last_table_flag = 1;
1545                goto end_function;
1546        }
1547
1548        /*
1549         * Calculate the data size of the next table.
1550         * Stop if no entries left or if data size is more the DMA restriction
1551         */
1552        next_table_data_size = 0;
1553        for (; counter < num_array_entries; counter++) {
1554                next_table_data_size += lli_in_array_ptr[counter].block_size;
1555                if (next_table_data_size >= SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE)
1556                        break;
1557        }
1558
1559        /*
1560         * Check if the next table data size is less then DMA rstriction.
1561         * if it is - recalculate the current table size, so that the next
1562         * table data size will be adaquete for DMA
1563         */
1564        if (next_table_data_size &&
1565                next_table_data_size < SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE)
1566
1567                table_data_size -= (SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE -
1568                        next_table_data_size);
1569
1570end_function:
1571        return table_data_size;
1572}
1573
1574/**
1575 * sep_build_lli_table - build an lli array for the given table
1576 * @sep: pointer to struct sep_device
1577 * @lli_array_ptr: pointer to lli array
1578 * @lli_table_ptr: pointer to lli table
1579 * @num_processed_entries_ptr: pointer to number of entries
1580 * @num_table_entries_ptr: pointer to number of tables
1581 * @table_data_size: total data size
1582 *
1583 * Builds an lli table from the lli_array according to
1584 * the given size of data
1585 */
1586static void sep_build_lli_table(struct sep_device *sep,
1587        struct sep_lli_entry    *lli_array_ptr,
1588        struct sep_lli_entry    *lli_table_ptr,
1589        u32 *num_processed_entries_ptr,
1590        u32 *num_table_entries_ptr,
1591        u32 table_data_size)
1592{
1593        /* Current table data size */
1594        u32 curr_table_data_size;
1595        /* Counter of lli array entry */
1596        u32 array_counter;
1597
1598        /* Init current table data size and lli array entry counter */
1599        curr_table_data_size = 0;
1600        array_counter = 0;
1601        *num_table_entries_ptr = 1;
1602
1603        dev_dbg(&sep->pdev->dev,
1604                "[PID%d] build lli table table_data_size: (hex) %x\n",
1605                        current->pid, table_data_size);
1606
1607        /* Fill the table till table size reaches the needed amount */
1608        while (curr_table_data_size < table_data_size) {
1609                /* Update the number of entries in table */
1610                (*num_table_entries_ptr)++;
1611
1612                lli_table_ptr->bus_address =
1613                        cpu_to_le32(lli_array_ptr[array_counter].bus_address);
1614
1615                lli_table_ptr->block_size =
1616                        cpu_to_le32(lli_array_ptr[array_counter].block_size);
1617
1618                curr_table_data_size += lli_array_ptr[array_counter].block_size;
1619
1620                dev_dbg(&sep->pdev->dev,
1621                        "[PID%d] lli_table_ptr is %p\n",
1622                                current->pid, lli_table_ptr);
1623                dev_dbg(&sep->pdev->dev,
1624                        "[PID%d] lli_table_ptr->bus_address: %08lx\n",
1625                                current->pid,
1626                                (unsigned long)lli_table_ptr->bus_address);
1627
1628                dev_dbg(&sep->pdev->dev,
1629                        "[PID%d] lli_table_ptr->block_size is (hex) %x\n",
1630                                current->pid, lli_table_ptr->block_size);
1631
1632                /* Check for overflow of the table data */
1633                if (curr_table_data_size > table_data_size) {
1634                        dev_dbg(&sep->pdev->dev,
1635                                "[PID%d] curr_table_data_size too large\n",
1636                                        current->pid);
1637
1638                        /* Update the size of block in the table */
1639                        lli_table_ptr->block_size =
1640                                cpu_to_le32(lli_table_ptr->block_size) -
1641                                (curr_table_data_size - table_data_size);
1642
1643                        /* Update the physical address in the lli array */
1644                        lli_array_ptr[array_counter].bus_address +=
1645                        cpu_to_le32(lli_table_ptr->block_size);
1646
1647                        /* Update the block size left in the lli array */
1648                        lli_array_ptr[array_counter].block_size =
1649                                (curr_table_data_size - table_data_size);
1650                } else
1651                        /* Advance to the next entry in the lli_array */
1652                        array_counter++;
1653
1654                dev_dbg(&sep->pdev->dev,
1655                        "[PID%d] lli_table_ptr->bus_address is %08lx\n",
1656                                current->pid,
1657                                (unsigned long)lli_table_ptr->bus_address);
1658                dev_dbg(&sep->pdev->dev,
1659                        "[PID%d] lli_table_ptr->block_size is (hex) %x\n",
1660                                current->pid,
1661                                lli_table_ptr->block_size);
1662
1663                /* Move to the next entry in table */
1664                lli_table_ptr++;
1665        }
1666
1667        /* Set the info entry to default */
1668        lli_table_ptr->bus_address = 0xffffffff;
1669        lli_table_ptr->block_size = 0;
1670
1671        /* Set the output parameter */
1672        *num_processed_entries_ptr += array_counter;
1673
1674}
1675
1676/**
1677 * sep_shared_area_virt_to_bus - map shared area to bus address
1678 * @sep: pointer to struct sep_device
1679 * @virt_address: virtual address to convert
1680 *
1681 * This functions returns the physical address inside shared area according
1682 * to the virtual address. It can be either on the external RAM device
1683 * (ioremapped), or on the system RAM
1684 * This implementation is for the external RAM
1685 */
1686static dma_addr_t sep_shared_area_virt_to_bus(struct sep_device *sep,
1687        void *virt_address)
1688{
1689        dev_dbg(&sep->pdev->dev, "[PID%d] sh virt to phys v %p\n",
1690                                        current->pid, virt_address);
1691        dev_dbg(&sep->pdev->dev, "[PID%d] sh virt to phys p %08lx\n",
1692                current->pid,
1693                (unsigned long)
1694                sep->shared_bus + (virt_address - sep->shared_addr));
1695
1696        return sep->shared_bus + (size_t)(virt_address - sep->shared_addr);
1697}
1698
1699/**
1700 * sep_shared_area_bus_to_virt - map shared area bus address to kernel
1701 * @sep: pointer to struct sep_device
1702 * @bus_address: bus address to convert
1703 *
1704 * This functions returns the virtual address inside shared area
1705 * according to the physical address. It can be either on the
1706 * external RAM device (ioremapped), or on the system RAM
1707 * This implementation is for the external RAM
1708 */
1709static void *sep_shared_area_bus_to_virt(struct sep_device *sep,
1710        dma_addr_t bus_address)
1711{
1712        dev_dbg(&sep->pdev->dev, "[PID%d] shared bus to virt b=%lx v=%lx\n",
1713                current->pid,
1714                (unsigned long)bus_address, (unsigned long)(sep->shared_addr +
1715                        (size_t)(bus_address - sep->shared_bus)));
1716
1717        return sep->shared_addr + (size_t)(bus_address - sep->shared_bus);
1718}
1719
1720/**
1721 * sep_debug_print_lli_tables - dump LLI table
1722 * @sep: pointer to struct sep_device
1723 * @lli_table_ptr: pointer to sep_lli_entry
1724 * @num_table_entries: number of entries
1725 * @table_data_size: total data size
1726 *
1727 * Walk the the list of the print created tables and print all the data
1728 */
1729static void sep_debug_print_lli_tables(struct sep_device *sep,
1730        struct sep_lli_entry *lli_table_ptr,
1731        unsigned long num_table_entries,
1732        unsigned long table_data_size)
1733{
1734#ifdef DEBUG
1735        unsigned long table_count = 1;
1736        unsigned long entries_count = 0;
1737
1738        dev_dbg(&sep->pdev->dev, "[PID%d] sep_debug_print_lli_tables start\n",
1739                                        current->pid);
1740        if (num_table_entries == 0) {
1741                dev_dbg(&sep->pdev->dev, "[PID%d] no table to print\n",
1742                        current->pid);
1743                return;
1744        }
1745
1746        while ((unsigned long) lli_table_ptr->bus_address != 0xffffffff) {
1747                dev_dbg(&sep->pdev->dev,
1748                        "[PID%d] lli table %08lx, "
1749                        "table_data_size is (hex) %lx\n",
1750                                current->pid, table_count, table_data_size);
1751                dev_dbg(&sep->pdev->dev,
1752                        "[PID%d] num_table_entries is (hex) %lx\n",
1753                                current->pid, num_table_entries);
1754
1755                /* Print entries of the table (without info entry) */
1756                for (entries_count = 0; entries_count < num_table_entries;
1757                        entries_count++, lli_table_ptr++) {
1758
1759                        dev_dbg(&sep->pdev->dev,
1760                                "[PID%d] lli_table_ptr address is %08lx\n",
1761                                current->pid,
1762                                (unsigned long) lli_table_ptr);
1763
1764                        dev_dbg(&sep->pdev->dev,
1765                                "[PID%d] phys address is %08lx "
1766                                "block size is (hex) %x\n", current->pid,
1767                                (unsigned long)lli_table_ptr->bus_address,
1768                                lli_table_ptr->block_size);
1769                }
1770
1771                /* Point to the info entry */
1772                lli_table_ptr--;
1773
1774                dev_dbg(&sep->pdev->dev,
1775                        "[PID%d] phys lli_table_ptr->block_size "
1776                        "is (hex) %x\n",
1777                        current->pid,
1778                        lli_table_ptr->block_size);
1779
1780                dev_dbg(&sep->pdev->dev,
1781                        "[PID%d] phys lli_table_ptr->physical_address "
1782                        "is %08lx\n",
1783                        current->pid,
1784                        (unsigned long)lli_table_ptr->bus_address);
1785
1786
1787                table_data_size = lli_table_ptr->block_size & 0xffffff;
1788                num_table_entries = (lli_table_ptr->block_size >> 24) & 0xff;
1789
1790                dev_dbg(&sep->pdev->dev,
1791                        "[PID%d] phys table_data_size is "
1792                        "(hex) %lx num_table_entries is"
1793                        " %lx bus_address is%lx\n",
1794                                current->pid,
1795                                table_data_size,
1796                                num_table_entries,
1797                                (unsigned long)lli_table_ptr->bus_address);
1798
1799                if ((unsigned long)lli_table_ptr->bus_address != 0xffffffff)
1800                        lli_table_ptr = (struct sep_lli_entry *)
1801                                sep_shared_bus_to_virt(sep,
1802                                (unsigned long)lli_table_ptr->bus_address);
1803
1804                table_count++;
1805        }
1806        dev_dbg(&sep->pdev->dev, "[PID%d] sep_debug_print_lli_tables end\n",
1807                                        current->pid);
1808#endif
1809}
1810
1811
1812/**
1813 * sep_prepare_empty_lli_table - create a blank LLI table
1814 * @sep: pointer to struct sep_device
1815 * @lli_table_addr_ptr: pointer to lli table
1816 * @num_entries_ptr: pointer to number of entries
1817 * @table_data_size_ptr: point to table data size
1818 * @dmatables_region: Optional buffer for DMA tables
1819 * @dma_ctx: DMA context
1820 *
1821 * This function creates empty lli tables when there is no data
1822 */
1823static void sep_prepare_empty_lli_table(struct sep_device *sep,
1824                dma_addr_t *lli_table_addr_ptr,
1825                u32 *num_entries_ptr,
1826                u32 *table_data_size_ptr,
1827                void **dmatables_region,
1828                struct sep_dma_context *dma_ctx)
1829{
1830        struct sep_lli_entry *lli_table_ptr;
1831
1832        /* Find the area for new table */
1833        lli_table_ptr =
1834                (struct sep_lli_entry *)(sep->shared_addr +
1835                SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES +
1836                dma_ctx->num_lli_tables_created * sizeof(struct sep_lli_entry) *
1837                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP);
1838
1839        if (dmatables_region && *dmatables_region)
1840                lli_table_ptr = *dmatables_region;
1841
1842        lli_table_ptr->bus_address = 0;
1843        lli_table_ptr->block_size = 0;
1844
1845        lli_table_ptr++;
1846        lli_table_ptr->bus_address = 0xFFFFFFFF;
1847        lli_table_ptr->block_size = 0;
1848
1849        /* Set the output parameter value */
1850        *lli_table_addr_ptr = sep->shared_bus +
1851                SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES +
1852                dma_ctx->num_lli_tables_created *
1853                sizeof(struct sep_lli_entry) *
1854                SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
1855
1856        /* Set the num of entries and table data size for empty table */
1857        *num_entries_ptr = 2;
1858        *table_data_size_ptr = 0;
1859
1860        /* Update the number of created tables */
1861        dma_ctx->num_lli_tables_created++;
1862}
1863
1864/**
1865 * sep_prepare_input_dma_table - prepare input DMA mappings
1866 * @sep: pointer to struct sep_device
1867 * @data_size:
1868 * @block_size:
1869 * @lli_table_ptr:
1870 * @num_entries_ptr:
1871 * @table_data_size_ptr:
1872 * @is_kva: set for kernel data (kernel crypt io call)
1873 *
1874 * This function prepares only input DMA table for synchronic symmetric
1875 * operations (HASH)
1876 * Note that all bus addresses that are passed to the SEP
1877 * are in 32 bit format; the SEP is a 32 bit device
1878 */
1879static int sep_prepare_input_dma_table(struct sep_device *sep,
1880        unsigned long app_virt_addr,
1881        u32 data_size,
1882        u32 block_size,
1883        dma_addr_t *lli_table_ptr,
1884        u32 *num_entries_ptr,
1885        u32 *table_data_size_ptr,
1886        bool is_kva,
1887        void **dmatables_region,
1888        struct sep_dma_context *dma_ctx
1889)
1890{
1891        int error = 0;
1892        /* Pointer to the info entry of the table - the last entry */
1893        struct sep_lli_entry *info_entry_ptr;
1894        /* Array of pointers to page */
1895        struct sep_lli_entry *lli_array_ptr;
1896        /* Points to the first entry to be processed in the lli_in_array */
1897        u32 current_entry = 0;
1898        /* Num entries in the virtual buffer */
1899        u32 sep_lli_entries = 0;
1900        /* Lli table pointer */
1901        struct sep_lli_entry *in_lli_table_ptr;
1902        /* The total data in one table */
1903        u32 table_data_size = 0;
1904        /* Flag for last table */
1905        u32 last_table_flag = 0;
1906        /* Number of entries in lli table */
1907        u32 num_entries_in_table = 0;
1908        /* Next table address */
1909        void *lli_table_alloc_addr = NULL;
1910        void *dma_lli_table_alloc_addr = NULL;
1911        void *dma_in_lli_table_ptr = NULL;
1912
1913        dev_dbg(&sep->pdev->dev,
1914                "[PID%d] prepare intput dma tbl data size: (hex) %x\n",
1915                current->pid, data_size);
1916
1917        dev_dbg(&sep->pdev->dev, "[PID%d] block_size is (hex) %x\n",
1918                                        current->pid, block_size);
1919
1920        /* Initialize the pages pointers */
1921        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL;
1922        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages = 0;
1923
1924        /* Set the kernel address for first table to be allocated */
1925        lli_table_alloc_addr = (void *)(sep->shared_addr +
1926                SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES +
1927                dma_ctx->num_lli_tables_created * sizeof(struct sep_lli_entry) *
1928                SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP);
1929
1930        if (data_size == 0) {
1931                if (dmatables_region) {
1932                        error = sep_allocate_dmatables_region(sep,
1933                                                dmatables_region,
1934                                                dma_ctx,
1935                                                1);
1936                        if (error)
1937                                return error;
1938                }
1939                /* Special case  - create meptu table - 2 entries, zero data */
1940                sep_prepare_empty_lli_table(sep, lli_table_ptr,
1941                                num_entries_ptr, table_data_size_ptr,
1942                                dmatables_region, dma_ctx);
1943                goto update_dcb_counter;
1944        }
1945
1946        /* Check if the pages are in Kernel Virtual Address layout */
1947        if (is_kva)
1948                error = sep_lock_kernel_pages(sep, app_virt_addr,
1949                        data_size, &lli_array_ptr, SEP_DRIVER_IN_FLAG,
1950                        dma_ctx);
1951        else
1952                /*
1953                 * Lock the pages of the user buffer
1954                 * and translate them to pages
1955                 */
1956                error = sep_lock_user_pages(sep, app_virt_addr,
1957                        data_size, &lli_array_ptr, SEP_DRIVER_IN_FLAG,
1958                        dma_ctx);
1959
1960        if (error)
1961                goto end_function;
1962
1963        dev_dbg(&sep->pdev->dev,
1964                "[PID%d] output sep_in_num_pages is (hex) %x\n",
1965                current->pid,
1966                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages);
1967
1968        current_entry = 0;
1969        info_entry_ptr = NULL;
1970
1971        sep_lli_entries =
1972                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages;
1973
1974        dma_lli_table_alloc_addr = lli_table_alloc_addr;
1975        if (dmatables_region) {
1976                error = sep_allocate_dmatables_region(sep,
1977                                        dmatables_region,
1978                                        dma_ctx,
1979                                        sep_lli_entries);
1980                if (error)
1981                        goto end_function_error;
1982                lli_table_alloc_addr = *dmatables_region;
1983        }
1984
1985        /* Loop till all the entries in in array are processed */
1986        while (current_entry < sep_lli_entries) {
1987
1988                /* Set the new input and output tables */
1989                in_lli_table_ptr =
1990                        (struct sep_lli_entry *)lli_table_alloc_addr;
1991                dma_in_lli_table_ptr =
1992                        (struct sep_lli_entry *)dma_lli_table_alloc_addr;
1993
1994                lli_table_alloc_addr += sizeof(struct sep_lli_entry) *
1995                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
1996                dma_lli_table_alloc_addr += sizeof(struct sep_lli_entry) *
1997                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
1998
1999                if (dma_lli_table_alloc_addr >
2000                        ((void *)sep->shared_addr +
2001                        SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES +
2002                        SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES)) {
2003
2004                        error = -ENOMEM;
2005                        goto end_function_error;
2006
2007                }
2008
2009                /* Update the number of created tables */
2010                dma_ctx->num_lli_tables_created++;
2011
2012                /* Calculate the maximum size of data for input table */
2013                table_data_size = sep_calculate_lli_table_max_size(sep,
2014                        &lli_array_ptr[current_entry],
2015                        (sep_lli_entries - current_entry),
2016                        &last_table_flag);
2017
2018                /*
2019                 * If this is not the last table -
2020                 * then align it to the block size
2021                 */
2022                if (!last_table_flag)
2023                        table_data_size =
2024                                (table_data_size / block_size) * block_size;
2025
2026                dev_dbg(&sep->pdev->dev,
2027                        "[PID%d] output table_data_size is (hex) %x\n",
2028                                current->pid,
2029                                table_data_size);
2030
2031                /* Construct input lli table */
2032                sep_build_lli_table(sep, &lli_array_ptr[current_entry],
2033                        in_lli_table_ptr,
2034                        &current_entry, &num_entries_in_table, table_data_size);
2035
2036                if (info_entry_ptr == NULL) {
2037
2038                        /* Set the output parameters to physical addresses */
2039                        *lli_table_ptr = sep_shared_area_virt_to_bus(sep,
2040                                dma_in_lli_table_ptr);
2041                        *num_entries_ptr = num_entries_in_table;
2042                        *table_data_size_ptr = table_data_size;
2043
2044                        dev_dbg(&sep->pdev->dev,
2045                                "[PID%d] output lli_table_in_ptr is %08lx\n",
2046                                current->pid,
2047                                (unsigned long)*lli_table_ptr);
2048
2049                } else {
2050                        /* Update the info entry of the previous in table */
2051                        info_entry_ptr->bus_address =
2052                                sep_shared_area_virt_to_bus(sep,
2053                                                        dma_in_lli_table_ptr);
2054                        info_entry_ptr->block_size =
2055                                ((num_entries_in_table) << 24) |
2056                                (table_data_size);
2057                }
2058                /* Save the pointer to the info entry of the current tables */
2059                info_entry_ptr = in_lli_table_ptr + num_entries_in_table - 1;
2060        }
2061        /* Print input tables */
2062        if (!dmatables_region) {
2063                sep_debug_print_lli_tables(sep, (struct sep_lli_entry *)
2064                        sep_shared_area_bus_to_virt(sep, *lli_table_ptr),
2065                        *num_entries_ptr, *table_data_size_ptr);
2066        }
2067
2068        /* The array of the pages */
2069        kfree(lli_array_ptr);
2070
2071update_dcb_counter:
2072        /* Update DCB counter */
2073        dma_ctx->nr_dcb_creat++;
2074        goto end_function;
2075
2076end_function_error:
2077        /* Free all the allocated resources */
2078        kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array);
2079        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array = NULL;
2080        kfree(lli_array_ptr);
2081        kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array);
2082        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL;
2083
2084end_function:
2085        return error;
2086
2087}
2088
2089/**
2090 * sep_construct_dma_tables_from_lli - prepare AES/DES mappings
2091 * @sep: pointer to struct sep_device
2092 * @lli_in_array:
2093 * @sep_in_lli_entries:
2094 * @lli_out_array:
2095 * @sep_out_lli_entries
2096 * @block_size
2097 * @lli_table_in_ptr
2098 * @lli_table_out_ptr
2099 * @in_num_entries_ptr
2100 * @out_num_entries_ptr
2101 * @table_data_size_ptr
2102 *
2103 * This function creates the input and output DMA tables for
2104 * symmetric operations (AES/DES) according to the block
2105 * size from LLI arays
2106 * Note that all bus addresses that are passed to the SEP
2107 * are in 32 bit format; the SEP is a 32 bit device
2108 */
2109static int sep_construct_dma_tables_from_lli(
2110        struct sep_device *sep,
2111        struct sep_lli_entry *lli_in_array,
2112        u32     sep_in_lli_entries,
2113        struct sep_lli_entry *lli_out_array,
2114        u32     sep_out_lli_entries,
2115        u32     block_size,
2116        dma_addr_t *lli_table_in_ptr,
2117        dma_addr_t *lli_table_out_ptr,
2118        u32     *in_num_entries_ptr,
2119        u32     *out_num_entries_ptr,
2120        u32     *table_data_size_ptr,
2121        void    **dmatables_region,
2122        struct sep_dma_context *dma_ctx)
2123{
2124        /* Points to the area where next lli table can be allocated */
2125        void *lli_table_alloc_addr = NULL;
2126        /*
2127         * Points to the area in shared region where next lli table
2128         * can be allocated
2129         */
2130        void *dma_lli_table_alloc_addr = NULL;
2131        /* Input lli table in dmatables_region or shared region */
2132        struct sep_lli_entry *in_lli_table_ptr = NULL;
2133        /* Input lli table location in the shared region */
2134        struct sep_lli_entry *dma_in_lli_table_ptr = NULL;
2135        /* Output lli table in dmatables_region or shared region */
2136        struct sep_lli_entry *out_lli_table_ptr = NULL;
2137        /* Output lli table location in the shared region */
2138        struct sep_lli_entry *dma_out_lli_table_ptr = NULL;
2139        /* Pointer to the info entry of the table - the last entry */
2140        struct sep_lli_entry *info_in_entry_ptr = NULL;
2141        /* Pointer to the info entry of the table - the last entry */
2142        struct sep_lli_entry *info_out_entry_ptr = NULL;
2143        /* Points to the first entry to be processed in the lli_in_array */
2144        u32 current_in_entry = 0;
2145        /* Points to the first entry to be processed in the lli_out_array */
2146        u32 current_out_entry = 0;
2147        /* Max size of the input table */
2148        u32 in_table_data_size = 0;
2149        /* Max size of the output table */
2150        u32 out_table_data_size = 0;
2151        /* Flag te signifies if this is the last tables build */
2152        u32 last_table_flag = 0;
2153        /* The data size that should be in table */
2154        u32 table_data_size = 0;
2155        /* Number of entries in the input table */
2156        u32 num_entries_in_table = 0;
2157        /* Number of entries in the output table */
2158        u32 num_entries_out_table = 0;
2159
2160        if (!dma_ctx) {
2161                dev_warn(&sep->pdev->dev, "DMA context uninitialized\n");
2162                return -EINVAL;
2163        }
2164
2165        /* Initiate to point after the message area */
2166        lli_table_alloc_addr = (void *)(sep->shared_addr +
2167                SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES +
2168                (dma_ctx->num_lli_tables_created *
2169                (sizeof(struct sep_lli_entry) *
2170                SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP)));
2171        dma_lli_table_alloc_addr = lli_table_alloc_addr;
2172
2173        if (dmatables_region) {
2174                /* 2 for both in+out table */
2175                if (sep_allocate_dmatables_region(sep,
2176                                        dmatables_region,
2177                                        dma_ctx,
2178                                        2*sep_in_lli_entries))
2179                        return -ENOMEM;
2180                lli_table_alloc_addr = *dmatables_region;
2181        }
2182
2183        /* Loop till all the entries in in array are not processed */
2184        while (current_in_entry < sep_in_lli_entries) {
2185                /* Set the new input and output tables */
2186                in_lli_table_ptr =
2187                        (struct sep_lli_entry *)lli_table_alloc_addr;
2188                dma_in_lli_table_ptr =
2189                        (struct sep_lli_entry *)dma_lli_table_alloc_addr;
2190
2191                lli_table_alloc_addr += sizeof(struct sep_lli_entry) *
2192                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
2193                dma_lli_table_alloc_addr += sizeof(struct sep_lli_entry) *
2194                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
2195
2196                /* Set the first output tables */
2197                out_lli_table_ptr =
2198                        (struct sep_lli_entry *)lli_table_alloc_addr;
2199                dma_out_lli_table_ptr =
2200                        (struct sep_lli_entry *)dma_lli_table_alloc_addr;
2201
2202                /* Check if the DMA table area limit was overrun */
2203                if ((dma_lli_table_alloc_addr + sizeof(struct sep_lli_entry) *
2204                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP) >
2205                        ((void *)sep->shared_addr +
2206                        SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES +
2207                        SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES)) {
2208
2209                        dev_warn(&sep->pdev->dev, "dma table limit overrun\n");
2210                        return -ENOMEM;
2211                }
2212
2213                /* Update the number of the lli tables created */
2214                dma_ctx->num_lli_tables_created += 2;
2215
2216                lli_table_alloc_addr += sizeof(struct sep_lli_entry) *
2217                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
2218                dma_lli_table_alloc_addr += sizeof(struct sep_lli_entry) *
2219                        SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP;
2220
2221                /* Calculate the maximum size of data for input table */
2222                in_table_data_size =
2223                        sep_calculate_lli_table_max_size(sep,
2224                        &lli_in_array[current_in_entry],
2225                        (sep_in_lli_entries - current_in_entry),
2226                        &last_table_flag);
2227
2228                /* Calculate the maximum size of data for output table */
2229                out_table_data_size =
2230                        sep_calculate_lli_table_max_size(sep,
2231                        &lli_out_array[current_out_entry],
2232                        (sep_out_lli_entries - current_out_entry),
2233                        &last_table_flag);
2234
2235                if (!last_table_flag) {
2236                        in_table_data_size = (in_table_data_size /
2237                                block_size) * block_size;
2238                        out_table_data_size = (out_table_data_size /
2239                                block_size) * block_size;
2240                }
2241
2242                table_data_size = in_table_data_size;
2243                if (table_data_size > out_table_data_size)
2244                        table_data_size = out_table_data_size;
2245
2246                dev_dbg(&sep->pdev->dev,
2247                        "[PID%d] construct tables from lli"
2248                        " in_table_data_size is (hex) %x\n", current->pid,
2249                        in_table_data_size);
2250
2251                dev_dbg(&sep->pdev->dev,
2252                        "[PID%d] construct tables from lli"
2253                        "out_table_data_size is (hex) %x\n", current->pid,
2254                        out_table_data_size);
2255
2256                /* Construct input lli table */
2257                sep_build_lli_table(sep, &lli_in_array[current_in_entry],
2258                        in_lli_table_ptr,
2259                        &current_in_entry,
2260                        &num_entries_in_table,
2261                        table_data_size);
2262
2263                /* Construct output lli table */
2264                sep_build_lli_table(sep, &lli_out_array[current_out_entry],
2265                        out_lli_table_ptr,
2266                        &current_out_entry,
2267                        &num_entries_out_table,
2268                        table_data_size);
2269
2270                /* If info entry is null - this is the first table built */
2271                if (info_in_entry_ptr == NULL || info_out_entry_ptr == NULL) {
2272                        /* Set the output parameters to physical addresses */
2273                        *lli_table_in_ptr =
2274                        sep_shared_area_virt_to_bus(sep, dma_in_lli_table_ptr);
2275
2276                        *in_num_entries_ptr = num_entries_in_table;
2277
2278                        *lli_table_out_ptr =
2279                                sep_shared_area_virt_to_bus(sep,
2280                                dma_out_lli_table_ptr);
2281
2282                        *out_num_entries_ptr = num_entries_out_table;
2283                        *table_data_size_ptr = table_data_size;
2284
2285                        dev_dbg(&sep->pdev->dev,
2286                                "[PID%d] output lli_table_in_ptr is %08lx\n",
2287                                current->pid,
2288                                (unsigned long)*lli_table_in_ptr);
2289                        dev_dbg(&sep->pdev->dev,
2290                                "[PID%d] output lli_table_out_ptr is %08lx\n",
2291                                current->pid,
2292                                (unsigned long)*lli_table_out_ptr);
2293                } else {
2294                        /* Update the info entry of the previous in table */
2295                        info_in_entry_ptr->bus_address =
2296                                sep_shared_area_virt_to_bus(sep,
2297                                dma_in_lli_table_ptr);
2298
2299                        info_in_entry_ptr->block_size =
2300                                ((num_entries_in_table) << 24) |
2301                                (table_data_size);
2302
2303                        /* Update the info entry of the previous in table */
2304                        info_out_entry_ptr->bus_address =
2305                                sep_shared_area_virt_to_bus(sep,
2306                                dma_out_lli_table_ptr);
2307
2308                        info_out_entry_ptr->block_size =
2309                                ((num_entries_out_table) << 24) |
2310                                (table_data_size);
2311
2312                        dev_dbg(&sep->pdev->dev,
2313                                "[PID%d] output lli_table_in_ptr:%08lx %08x\n",
2314                                current->pid,
2315                                (unsigned long)info_in_entry_ptr->bus_address,
2316                                info_in_entry_ptr->block_size);
2317
2318                        dev_dbg(&sep->pdev->dev,
2319                                "[PID%d] output lli_table_out_ptr:"
2320                                "%08lx  %08x\n",
2321                                current->pid,
2322                                (unsigned long)info_out_entry_ptr->bus_address,
2323                                info_out_entry_ptr->block_size);
2324                }
2325
2326                /* Save the pointer to the info entry of the current tables */
2327                info_in_entry_ptr = in_lli_table_ptr +
2328                        num_entries_in_table - 1;
2329                info_out_entry_ptr = out_lli_table_ptr +
2330                        num_entries_out_table - 1;
2331
2332                dev_dbg(&sep->pdev->dev,
2333                        "[PID%d] output num_entries_out_table is %x\n",
2334                        current->pid,
2335                        (u32)num_entries_out_table);
2336                dev_dbg(&sep->pdev->dev,
2337                        "[PID%d] output info_in_entry_ptr is %lx\n",
2338                        current->pid,
2339                        (unsigned long)info_in_entry_ptr);
2340                dev_dbg(&sep->pdev->dev,
2341                        "[PID%d] output info_out_entry_ptr is %lx\n",
2342                        current->pid,
2343                        (unsigned long)info_out_entry_ptr);
2344        }
2345
2346        /* Print input tables */
2347        if (!dmatables_region) {
2348                sep_debug_print_lli_tables(
2349                        sep,
2350                        (struct sep_lli_entry *)
2351                        sep_shared_area_bus_to_virt(sep, *lli_table_in_ptr),
2352                        *in_num_entries_ptr,
2353                        *table_data_size_ptr);
2354        }
2355
2356        /* Print output tables */
2357        if (!dmatables_region) {
2358                sep_debug_print_lli_tables(
2359                        sep,
2360                        (struct sep_lli_entry *)
2361                        sep_shared_area_bus_to_virt(sep, *lli_table_out_ptr),
2362                        *out_num_entries_ptr,
2363                        *table_data_size_ptr);
2364        }
2365
2366        return 0;
2367}
2368
2369/**
2370 * sep_prepare_input_output_dma_table - prepare DMA I/O table
2371 * @app_virt_in_addr:
2372 * @app_virt_out_addr:
2373 * @data_size:
2374 * @block_size:
2375 * @lli_table_in_ptr:
2376 * @lli_table_out_ptr:
2377 * @in_num_entries_ptr:
2378 * @out_num_entries_ptr:
2379 * @table_data_size_ptr:
2380 * @is_kva: set for kernel data; used only for kernel crypto module
2381 *
2382 * This function builds input and output DMA tables for synchronic
2383 * symmetric operations (AES, DES, HASH). It also checks that each table
2384 * is of the modular block size
2385 * Note that all bus addresses that are passed to the SEP
2386 * are in 32 bit format; the SEP is a 32 bit device
2387 */
2388static int sep_prepare_input_output_dma_table(struct sep_device *sep,
2389        unsigned long app_virt_in_addr,
2390        unsigned long app_virt_out_addr,
2391        u32 data_size,
2392        u32 block_size,
2393        dma_addr_t *lli_table_in_ptr,
2394        dma_addr_t *lli_table_out_ptr,
2395        u32 *in_num_entries_ptr,
2396        u32 *out_num_entries_ptr,
2397        u32 *table_data_size_ptr,
2398        bool is_kva,
2399        void **dmatables_region,
2400        struct sep_dma_context *dma_ctx)
2401
2402{
2403        int error = 0;
2404        /* Array of pointers of page */
2405        struct sep_lli_entry *lli_in_array;
2406        /* Array of pointers of page */
2407        struct sep_lli_entry *lli_out_array;
2408
2409        if (!dma_ctx) {
2410                error = -EINVAL;
2411                goto end_function;
2412        }
2413
2414        if (data_size == 0) {
2415                /* Prepare empty table for input and output */
2416                if (dmatables_region) {
2417                        error = sep_allocate_dmatables_region(
2418                                        sep,
2419                                        dmatables_region,
2420                                        dma_ctx,
2421                                        2);
2422                  if (error)
2423                        goto end_function;
2424                }
2425                sep_prepare_empty_lli_table(sep, lli_table_in_ptr,
2426                        in_num_entries_ptr, table_data_size_ptr,
2427                        dmatables_region, dma_ctx);
2428
2429                sep_prepare_empty_lli_table(sep, lli_table_out_ptr,
2430                        out_num_entries_ptr, table_data_size_ptr,
2431                        dmatables_region, dma_ctx);
2432
2433                goto update_dcb_counter;
2434        }
2435
2436        /* Initialize the pages pointers */
2437        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL;
2438        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = NULL;
2439
2440        /* Lock the pages of the buffer and translate them to pages */
2441        if (is_kva) {
2442                dev_dbg(&sep->pdev->dev, "[PID%d] Locking kernel input pages\n",
2443                                                current->pid);
2444                error = sep_lock_kernel_pages(sep, app_virt_in_addr,
2445                                data_size, &lli_in_array, SEP_DRIVER_IN_FLAG,
2446                                dma_ctx);
2447                if (error) {
2448                        dev_warn(&sep->pdev->dev,
2449                                "[PID%d] sep_lock_kernel_pages for input "
2450                                "virtual buffer failed\n", current->pid);
2451
2452                        goto end_function;
2453                }
2454
2455                dev_dbg(&sep->pdev->dev, "[PID%d] Locking kernel output pages\n",
2456                                                current->pid);
2457                error = sep_lock_kernel_pages(sep, app_virt_out_addr,
2458                                data_size, &lli_out_array, SEP_DRIVER_OUT_FLAG,
2459                                dma_ctx);
2460
2461                if (error) {
2462                        dev_warn(&sep->pdev->dev,
2463                                "[PID%d] sep_lock_kernel_pages for output "
2464                                "virtual buffer failed\n", current->pid);
2465
2466                        goto end_function_free_lli_in;
2467                }
2468
2469        }
2470
2471        else {
2472                dev_dbg(&sep->pdev->dev, "[PID%d] Locking user input pages\n",
2473                                                current->pid);
2474                error = sep_lock_user_pages(sep, app_virt_in_addr,
2475                                data_size, &lli_in_array, SEP_DRIVER_IN_FLAG,
2476                                dma_ctx);
2477                if (error) {
2478                        dev_warn(&sep->pdev->dev,
2479                                "[PID%d] sep_lock_user_pages for input "
2480                                "virtual buffer failed\n", current->pid);
2481
2482                        goto end_function;
2483                }
2484
2485                if (dma_ctx->secure_dma) {
2486                        /* secure_dma requires use of non accessible memory */
2487                        dev_dbg(&sep->pdev->dev, "[PID%d] in secure_dma\n",
2488                                current->pid);
2489                        error = sep_lli_table_secure_dma(sep,
2490                                app_virt_out_addr, data_size, &lli_out_array,
2491                                SEP_DRIVER_OUT_FLAG, dma_ctx);
2492                        if (error) {
2493                                dev_warn(&sep->pdev->dev,
2494                                        "[PID%d] secure dma table setup "
2495                                        " for output virtual buffer failed\n",
2496                                        current->pid);
2497
2498                                goto end_function_free_lli_in;
2499                        }
2500                } else {
2501                        /* For normal, non-secure dma */
2502                        dev_dbg(&sep->pdev->dev, "[PID%d] not in secure_dma\n",
2503                                current->pid);
2504
2505                        dev_dbg(&sep->pdev->dev,
2506                                "[PID%d] Locking user output pages\n",
2507                                current->pid);
2508
2509                        error = sep_lock_user_pages(sep, app_virt_out_addr,
2510                                data_size, &lli_out_array, SEP_DRIVER_OUT_FLAG,
2511                                dma_ctx);
2512
2513                        if (error) {
2514                                dev_warn(&sep->pdev->dev,
2515                                        "[PID%d] sep_lock_user_pages"
2516                                        " for output virtual buffer failed\n",
2517                                        current->pid);
2518
2519                                goto end_function_free_lli_in;
2520                        }
2521                }
2522        }
2523
2524        dev_dbg(&sep->pdev->dev,
2525                "[PID%d] After lock; prep input output dma table sep_in_num_pages is (hex) %x\n",
2526                current->pid,
2527                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_num_pages);
2528
2529        dev_dbg(&sep->pdev->dev, "[PID%d] sep_out_num_pages is (hex) %x\n",
2530                current->pid,
2531                dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_num_pages);
2532
2533        dev_dbg(&sep->pdev->dev,
2534                "[PID%d] SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP is (hex) %x\n",
2535                current->pid, SEP_DRIVER_ENTRIES_PER_TABLE_IN_SEP);
2536
2537        /* Call the function that creates table from the lli arrays */
2538        dev_dbg(&sep->pdev->dev, "[PID%d] calling create table from lli\n",
2539                                        current->pid);
2540        error = sep_construct_dma_tables_from_lli(
2541                        sep, lli_in_array,
2542                        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].
2543                                                                in_num_pages,
2544                        lli_out_array,
2545                        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].
2546                                                                out_num_pages,
2547                        block_size, lli_table_in_ptr, lli_table_out_ptr,
2548                        in_num_entries_ptr, out_num_entries_ptr,
2549                        table_data_size_ptr, dmatables_region, dma_ctx);
2550
2551        if (error) {
2552                dev_warn(&sep->pdev->dev,
2553                        "[PID%d] sep_construct_dma_tables_from_lli failed\n",
2554                        current->pid);
2555                goto end_function_with_error;
2556        }
2557
2558        kfree(lli_out_array);
2559        kfree(lli_in_array);
2560
2561update_dcb_counter:
2562        /* Update DCB counter */
2563        dma_ctx->nr_dcb_creat++;
2564
2565        goto end_function;
2566
2567end_function_with_error:
2568        kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array);
2569        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_map_array = NULL;
2570        kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array);
2571        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].out_page_array = NULL;
2572        kfree(lli_out_array);
2573
2574
2575end_function_free_lli_in:
2576        kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array);
2577        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_map_array = NULL;
2578        kfree(dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array);
2579        dma_ctx->dma_res_arr[dma_ctx->nr_dcb_creat].in_page_array = NULL;
2580        kfree(lli_in_array);
2581
2582end_function:
2583
2584        return error;
2585
2586}
2587
2588/**
2589 * sep_prepare_input_output_dma_table_in_dcb - prepare control blocks
2590 * @app_in_address: unsigned long; for data buffer in (user space)
2591 * @app_out_address: unsigned long; for data buffer out (user space)
2592 * @data_in_size: u32; for size of data
2593 * @block_size: u32; for block size
2594 * @tail_block_size: u32; for size of tail block
2595 * @isapplet: bool; to indicate external app
2596 * @is_kva: bool; kernel buffer; only used for kernel crypto module
2597 * @secure_dma; indicates whether this is secure_dma using IMR
2598 *
2599 * This function prepares the linked DMA tables and puts the
2600 * address for the linked list of tables inta a DCB (data control
2601 * block) the address of which is known by the SEP hardware
2602 * Note that all bus addresses that are passed to the SEP
2603 * are in 32 bit format; the SEP is a 32 bit device
2604 */
2605int sep_prepare_input_output_dma_table_in_dcb(struct sep_device *sep,
2606        unsigned long  app_in_address,
2607        unsigned long  app_out_address,
2608        u32  data_in_size,
2609        u32  block_size,
2610        u32  tail_block_size,
2611        bool isapplet,
2612        bool    is_kva,
2613        bool    secure_dma,
2614        struct sep_dcblock *dcb_region,
2615        void **dmatables_region,
2616        struct sep_dma_context **dma_ctx,
2617        struct scatterlist *src_sg,
2618        struct scatterlist *dst_sg)
2619{
2620        int error = 0;
2621        /* Size of tail */
2622        u32 tail_size = 0;
2623        /* Address of the created DCB table */
2624        struct sep_dcblock *dcb_table_ptr = NULL;
2625        /* The physical address of the first input DMA table */
2626        dma_addr_t in_first_mlli_address = 0;
2627        /* Number of entries in the first input DMA table */
2628        u32  in_first_num_entries = 0;
2629        /* The physical address of the first output DMA table */
2630        dma_addr_t  out_first_mlli_address = 0;
2631        /* Number of entries in the first output DMA table */
2632        u32  out_first_num_entries = 0;
2633        /* Data in the first input/output table */
2634        u32  first_data_size = 0;
2635
2636        dev_dbg(&sep->pdev->dev, "[PID%d] app_in_address %lx\n",
2637                current->pid, app_in_address);
2638
2639        dev_dbg(&sep->pdev->dev, "[PID%d] app_out_address %lx\n",
2640                current->pid, app_out_address);
2641
2642        dev_dbg(&sep->pdev->dev, "[PID%d] data_in_size %x\n",
2643                current->pid, data_in_size);
2644
2645        dev_dbg(&sep->pdev->dev, "[PID%d] block_size %x\n",
2646                current->pid, block_size);
2647
2648        dev_dbg(&sep->pdev->dev, "[PID%d] tail_block_size %x\n",
2649                current->pid, tail_block_size);
2650
2651        dev_dbg(&sep->pdev->dev, "[PID%d] isapplet %x\n",
2652                current->pid, isapplet);
2653
2654        dev_dbg(&sep->pdev->dev, "[PID%d] is_kva %x\n",
2655                current->pid, is_kva);
2656
2657        dev_dbg(&sep->pdev->dev, "[PID%d] src_sg %p\n",
2658                current->pid, src_sg);
2659
2660        dev_dbg(&sep->pdev->dev, "[PID%d] dst_sg %p\n",
2661                current->pid, dst_sg);
2662
2663        if (!dma_ctx) {
2664                dev_warn(&sep->pdev->dev, "[PID%d] no DMA context pointer\n",
2665                                                current->pid);
2666                error = -EINVAL;
2667                goto end_function;
2668        }
2669
2670        if (*dma_ctx) {
2671                /* In case there are multiple DCBs for this transaction */
2672                dev_dbg(&sep->pdev->dev, "[PID%d] DMA context already set\n",
2673                                                current->pid);
2674        } else {
2675                *dma_ctx = kzalloc(sizeof(**dma_ctx), GFP_KERNEL);
2676                if (!(*dma_ctx)) {
2677                        dev_dbg(&sep->pdev->dev,
2678                                "[PID%d] Not enough memory for DMA context\n",
2679                                current->pid);
2680                  error = -ENOMEM;
2681                  goto end_function;
2682                }
2683                dev_dbg(&sep->pdev->dev,
2684                        "[PID%d] Created DMA context addr at 0x%p\n",
2685                        current->pid, *dma_ctx);
2686        }
2687
2688        (*dma_ctx)->secure_dma = secure_dma;
2689
2690        /* these are for kernel crypto only */
2691        (*dma_ctx)->src_sg = src_sg;
2692        (*dma_ctx)->dst_sg = dst_sg;
2693
2694        if ((*dma_ctx)->nr_dcb_creat == SEP_MAX_NUM_SYNC_DMA_OPS) {
2695                /* No more DCBs to allocate */
2696                dev_dbg(&sep->pdev->dev, "[PID%d] no more DCBs available\n",
2697                                                current->pid);
2698                error = -ENOSPC;
2699                goto end_function_error;
2700        }
2701
2702        /* Allocate new DCB */
2703        if (dcb_region) {
2704                dcb_table_ptr = dcb_region;
2705        } else {
2706                dcb_table_ptr = (struct sep_dcblock *)(sep->shared_addr +
2707                        SEP_DRIVER_SYSTEM_DCB_MEMORY_OFFSET_IN_BYTES +
2708                        ((*dma_ctx)->nr_dcb_creat *
2709                                                sizeof(struct sep_dcblock)));
2710        }
2711
2712        /* Set the default values in the DCB */
2713        dcb_table_ptr->input_mlli_address = 0;
2714        dcb_table_ptr->input_mlli_num_entries = 0;
2715        dcb_table_ptr->input_mlli_data_size = 0;
2716        dcb_table_ptr->output_mlli_address = 0;
2717        dcb_table_ptr->output_mlli_num_entries = 0;
2718        dcb_table_ptr->output_mlli_data_size = 0;
2719        dcb_table_ptr->tail_data_size = 0;
2720        dcb_table_ptr->out_vr_tail_pt = 0;
2721
2722        if (isapplet) {
2723
2724                /* Check if there is enough data for DMA operation */
2725                if (data_in_size < SEP_DRIVER_MIN_DATA_SIZE_PER_TABLE) {
2726                        if (is_kva) {
2727                                error = -ENODEV;
2728                                goto end_function_error;
2729                        } else {
2730                                if (copy_from_user(dcb_table_ptr->tail_data,
2731                                        (void __user *)app_in_address,
2732                                        data_in_size)) {
2733                                        error = -EFAULT;
2734                                        goto end_function_error;
2735                                }
2736                        }
2737
2738                        dcb_table_ptr->tail_data_size = data_in_size;
2739
2740                        /* Set the output user-space address for mem2mem op */
2741                        if (app_out_address)
2742                                dcb_table_ptr->out_vr_tail_pt =
2743                                (aligned_u64)app_out_address;
2744
2745                        /*
2746                         * Update both data length parameters in order to avoid
2747                         * second data copy and allow building of empty mlli
2748                         * tables
2749                         */
2750                        tail_size = 0x0;
2751                        data_in_size = 0x0;
2752
2753                } else {
2754                        if (!app_out_address) {
2755                                tail_size = data_in_size % block_size;
2756                                if (!tail_size) {
2757                                        if (tail_block_size == block_size)
2758                                                tail_size = block_size;
2759                                }
2760                        } else {
2761                                tail_size = 0;
2762                        }
2763                }
2764                if (tail_size) {
2765                        if (tail_size > sizeof(dcb_table_ptr->tail_data))
2766                                return -EINVAL;
2767                        if (is_kva) {
2768                                error = -ENODEV;
2769                                goto end_function_error;
2770                        } else {
2771                                /* We have tail data - copy it to DCB */
2772                                if (copy_from_user(dcb_table_ptr->tail_data,
2773                                        (void __user *)(app_in_address +
2774                                        data_in_size - tail_size), tail_size)) {
2775                                        error = -EFAULT;
2776                                        goto end_function_error;
2777                                }
2778                        }
2779                        if (app_out_address)
2780                                /*
2781                                 * Calculate the output address
2782                                 * according to tail data size
2783                                 */
2784                                dcb_table_ptr->out_vr_tail_pt =
2785                                        (aligned_u64)app_out_address +
2786                                        data_in_size - tail_size;
2787
2788                        /* Save the real tail data size */
2789                        dcb_table_ptr->tail_data_size = tail_size;
2790                        /*
2791                         * Update the data size without the tail
2792                         * data size AKA data for the dma
2793                         */
2794                        data_in_size = (data_in_size - tail_size);
2795                }
2796        }
2797        /* Check if we need to build only input table or input/output */
2798        if (app_out_address) {
2799                /* Prepare input/output tables */
2800                error = sep_prepare_input_output_dma_table(sep,
2801                                app_in_address,
2802                                app_out_address,
2803                                data_in_size,
2804                                block_size,
2805                                &in_first_mlli_address,
2806                                &out_first_mlli_address,
2807                                &in_first_num_entries,
2808                                &out_first_num_entries,
2809                                &first_data_size,
2810                                is_kva,
2811                                dmatables_region,
2812                                *dma_ctx);
2813        } else {
2814                /* Prepare input tables */
2815                error = sep_prepare_input_dma_table(sep,
2816                                app_in_address,
2817                                data_in_size,
2818                                block_size,
2819                                &in_first_mlli_address,
2820                                &in_first_num_entries,
2821                                &first_data_size,
2822                                is_kva,
2823                                dmatables_region,
2824                                *dma_ctx);
2825        }
2826
2827        if (error) {
2828                dev_warn(&sep->pdev->dev,
2829                        "prepare DMA table call failed "
2830                        "from prepare DCB call\n");
2831                goto end_function_error;
2832        }
2833
2834        /* Set the DCB values */
2835        dcb_table_ptr->input_mlli_address = in_first_mlli_address;
2836        dcb_table_ptr->input_mlli_num_entries = in_first_num_entries;
2837        dcb_table_ptr->input_mlli_data_size = first_data_size;
2838        dcb_table_ptr->output_mlli_address = out_first_mlli_address;
2839        dcb_table_ptr->output_mlli_num_entries = out_first_num_entries;
2840        dcb_table_ptr->output_mlli_data_size = first_data_size;
2841
2842        goto end_function;
2843
2844end_function_error:
2845        kfree(*dma_ctx);
2846        *dma_ctx = NULL;
2847
2848end_function:
2849        return error;
2850
2851}
2852
2853
2854/**
2855 * sep_free_dma_tables_and_dcb - free DMA tables and DCBs
2856 * @sep: pointer to struct sep_device
2857 * @isapplet: indicates external application (used for kernel access)
2858 * @is_kva: indicates kernel addresses (only used for kernel crypto)
2859 *
2860 * This function frees the DMA tables and DCB
2861 */
2862static int sep_free_dma_tables_and_dcb(struct sep_device *sep, bool isapplet,
2863        bool is_kva, struct sep_dma_context **dma_ctx)
2864{
2865        struct sep_dcblock *dcb_table_ptr;
2866        unsigned long pt_hold;
2867        void *tail_pt;
2868
2869        int i = 0;
2870        int error = 0;
2871        int error_temp = 0;
2872
2873        dev_dbg(&sep->pdev->dev, "[PID%d] sep_free_dma_tables_and_dcb\n",
2874                                        current->pid);
2875        if (!dma_ctx || !*dma_ctx) /* nothing to be done here*/
2876                return 0;
2877
2878        if (!(*dma_ctx)->secure_dma && isapplet) {
2879                dev_dbg(&sep->pdev->dev, "[PID%d] handling applet\n",
2880                        current->pid);
2881
2882                /* Tail stuff is only for non secure_dma */
2883                /* Set pointer to first DCB table */
2884                dcb_table_ptr = (struct sep_dcblock *)
2885                        (sep->shared_addr +
2886                        SEP_DRIVER_SYSTEM_DCB_MEMORY_OFFSET_IN_BYTES);
2887
2888                /**
2889                 * Go over each DCB and see if
2890                 * tail pointer must be updated
2891                 */
2892                for (i = 0; i < (*dma_ctx)->nr_dcb_creat; i++, dcb_table_ptr++) {
2893                        if (dcb_table_ptr->out_vr_tail_pt) {
2894                                pt_hold = (unsigned long)dcb_table_ptr->
2895                                        out_vr_tail_pt;
2896                                tail_pt = (void *)pt_hold;
2897                                if (is_kva) {
2898                                        error = -ENODEV;
2899                                        break;
2900                                } else {
2901                                        error_temp = copy_to_user(
2902                                                (void __user *)tail_pt,
2903                                                dcb_table_ptr->tail_data,
2904                                                dcb_table_ptr->tail_data_size);
2905                                }
2906                                if (error_temp) {
2907                                        /* Release the DMA resource */
2908                                        error = -EFAULT;
2909                                        break;
2910                                }
2911                        }
2912                }
2913        }
2914
2915        /* Free the output pages, if any */
2916        sep_free_dma_table_data_handler(sep, dma_ctx);
2917
2918        dev_dbg(&sep->pdev->dev, "[PID%d] sep_free_dma_tables_and_dcb end\n",
2919                                        current->pid);
2920
2921        return error;
2922}
2923
2924/**
2925 * sep_prepare_dcb_handler - prepare a control block
2926 * @sep: pointer to struct sep_device
2927 * @arg: pointer to user parameters
2928 * @secure_dma: indicate whether we are using secure_dma on IMR
2929 *
2930 * This function will retrieve the RAR buffer physical addresses, type
2931 * & size corresponding to the RAR handles provided in the buffers vector.
2932 */
2933static int sep_prepare_dcb_handler(struct sep_device *sep, unsigned long arg,
2934                                   bool secure_dma,
2935                                   struct sep_dma_context **dma_ctx)
2936{
2937        int error;
2938        /* Command arguments */
2939        static struct build_dcb_struct command_args;
2940
2941        /* Get the command arguments */
2942        if (copy_from_user(&command_args, (void __user *)arg,
2943                                        sizeof(struct build_dcb_struct))) {
2944                error = -EFAULT;
2945                goto end_function;
2946        }
2947
2948        dev_dbg(&sep->pdev->dev,
2949                "[PID%d] prep dcb handler app_in_address is %08llx\n",
2950                        current->pid, command_args.app_in_address);
2951        dev_dbg(&sep->pdev->dev,
2952                "[PID%d] app_out_address is %08llx\n",
2953                        current->pid, command_args.app_out_address);
2954        dev_dbg(&sep->pdev->dev,
2955                "[PID%d] data_size is %x\n",
2956                        current->pid, command_args.data_in_size);
2957        dev_dbg(&sep->pdev->dev,
2958                "[PID%d] block_size is %x\n",
2959                        current->pid, command_args.block_size);
2960        dev_dbg(&sep->pdev->dev,
2961                "[PID%d] tail block_size is %x\n",
2962                        current->pid, command_args.tail_block_size);
2963        dev_dbg(&sep->pdev->dev,
2964                "[PID%d] is_applet is %x\n",
2965                        current->pid, command_args.is_applet);
2966
2967        if (!command_args.app_in_address) {
2968                dev_warn(&sep->pdev->dev,
2969                        "[PID%d] null app_in_address\n", current->pid);
2970                error = -EINVAL;
2971                goto end_function;
2972        }
2973
2974        error = sep_prepare_input_output_dma_table_in_dcb(sep,
2975                        (unsigned long)command_args.app_in_address,
2976                        (unsigned long)command_args.app_out_address,
2977                        command_args.data_in_size, command_args.block_size,
2978                        command_args.tail_block_size,
2979                        command_args.is_applet, false,
2980                        secure_dma, NULL, NULL, dma_ctx, NULL, NULL);
2981
2982end_function:
2983        return error;
2984
2985}
2986
2987/**
2988 * sep_free_dcb_handler - free control block resources
2989 * @sep: pointer to struct sep_device
2990 *
2991 * This function frees the DCB resources and updates the needed
2992 * user-space buffers.
2993 */
2994static int sep_free_dcb_handler(struct sep_device *sep,
2995                                struct sep_dma_context **dma_ctx)
2996{
2997        if (!dma_ctx || !(*dma_ctx)) {
2998                dev_dbg(&sep->pdev->dev,
2999                        "[PID%d] no dma context defined, nothing to free\n",
3000                        current->pid);
3001                return -EINVAL;
3002        }
3003
3004        dev_dbg(&sep->pdev->dev, "[PID%d] free dcbs num of DCBs %x\n",
3005                current->pid,
3006                (*dma_ctx)->nr_dcb_creat);
3007
3008        return sep_free_dma_tables_and_dcb(sep, false, false, dma_ctx);
3009}
3010
3011/**
3012 * sep_ioctl - ioctl handler for sep device
3013 * @filp: pointer to struct file
3014 * @cmd: command
3015 * @arg: pointer to argument structure
3016 *
3017 * Implement the ioctl methods available on the SEP device.
3018 */
3019static long sep_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3020{
3021        struct sep_private_data * const private_data = filp->private_data;
3022        struct sep_call_status *call_status = &private_data->call_status;
3023        struct sep_device *sep = private_data->device;
3024        struct sep_dma_context **dma_ctx = &private_data->dma_ctx;
3025        struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem;
3026        int error = 0;
3027
3028        dev_dbg(&sep->pdev->dev, "[PID%d] ioctl cmd 0x%x\n",
3029                current->pid, cmd);
3030        dev_dbg(&sep->pdev->dev, "[PID%d] dma context addr 0x%p\n",
3031                current->pid, *dma_ctx);
3032
3033        /* Make sure we own this device */
3034        error = sep_check_transaction_owner(sep);
3035        if (error) {
3036                dev_dbg(&sep->pdev->dev, "[PID%d] ioctl pid is not owner\n",
3037                        current->pid);
3038                goto end_function;
3039        }
3040
3041        /* Check that sep_mmap has been called before */
3042        if (0 == test_bit(SEP_LEGACY_MMAP_DONE_OFFSET,
3043                                &call_status->status)) {
3044                dev_dbg(&sep->pdev->dev,
3045                        "[PID%d] mmap not called\n", current->pid);
3046                error = -EPROTO;
3047                goto end_function;
3048        }
3049
3050        /* Check that the command is for SEP device */
3051        if (_IOC_TYPE(cmd) != SEP_IOC_MAGIC_NUMBER) {
3052                error = -ENOTTY;
3053                goto end_function;
3054        }
3055
3056        switch (cmd) {
3057        case SEP_IOCSENDSEPCOMMAND:
3058                dev_dbg(&sep->pdev->dev,
3059                        "[PID%d] SEP_IOCSENDSEPCOMMAND start\n",
3060                        current->pid);
3061                if (1 == test_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET,
3062                                  &call_status->status)) {
3063                        dev_warn(&sep->pdev->dev,
3064                                "[PID%d] send msg already done\n",
3065                                current->pid);
3066                        error = -EPROTO;
3067                        goto end_function;
3068                }
3069                /* Send command to SEP */
3070                error = sep_send_command_handler(sep);
3071                if (!error)
3072                        set_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET,
3073                                &call_status->status);
3074                dev_dbg(&sep->pdev->dev,
3075                        "[PID%d] SEP_IOCSENDSEPCOMMAND end\n",
3076                        current->pid);
3077                break;
3078        case SEP_IOCENDTRANSACTION:
3079                dev_dbg(&sep->pdev->dev,
3080                        "[PID%d] SEP_IOCENDTRANSACTION start\n",
3081                        current->pid);
3082                error = sep_end_transaction_handler(sep, dma_ctx, call_status,
3083                                                    my_queue_elem);
3084                dev_dbg(&sep->pdev->dev,
3085                        "[PID%d] SEP_IOCENDTRANSACTION end\n",
3086                        current->pid);
3087                break;
3088        case SEP_IOCPREPAREDCB:
3089                dev_dbg(&sep->pdev->dev,
3090                        "[PID%d] SEP_IOCPREPAREDCB start\n",
3091                        current->pid);
3092        case SEP_IOCPREPAREDCB_SECURE_DMA:
3093                dev_dbg(&sep->pdev->dev,
3094                        "[PID%d] SEP_IOCPREPAREDCB_SECURE_DMA start\n",
3095                        current->pid);
3096                if (1 == test_bit(SEP_LEGACY_SENDMSG_DONE_OFFSET,
3097                                  &call_status->status)) {
3098                        dev_dbg(&sep->pdev->dev,
3099                                "[PID%d] dcb prep needed before send msg\n",
3100                                current->pid);
3101                        error = -EPROTO;
3102                        goto end_function;
3103                }
3104
3105                if (!arg) {
3106                        dev_dbg(&sep->pdev->dev,
3107                                "[PID%d] dcb null arg\n", current->pid);
3108                        error = -EINVAL;
3109                        goto end_function;
3110                }
3111
3112                if (cmd == SEP_IOCPREPAREDCB) {
3113                        /* No secure dma */
3114                        dev_dbg(&sep->pdev->dev,
3115                                "[PID%d] SEP_IOCPREPAREDCB (no secure_dma)\n",
3116                                current->pid);
3117
3118                        error = sep_prepare_dcb_handler(sep, arg, false,
3119                                dma_ctx);
3120                } else {
3121                        /* Secure dma */
3122                        dev_dbg(&sep->pdev->dev,
3123                                "[PID%d] SEP_IOC_POC (with secure_dma)\n",
3124                                current->pid);
3125
3126                        error = sep_prepare_dcb_handler(sep, arg, true,
3127                                dma_ctx);
3128                }
3129                dev_dbg(&sep->pdev->dev, "[PID%d] dcb's end\n",
3130                        current->pid);
3131                break;
3132        case SEP_IOCFREEDCB:
3133                dev_dbg(&sep->pdev->dev, "[PID%d] SEP_IOCFREEDCB start\n",
3134                        current->pid);
3135        case SEP_IOCFREEDCB_SECURE_DMA:
3136                dev_dbg(&sep->pdev->dev,
3137                        "[PID%d] SEP_IOCFREEDCB_SECURE_DMA start\n",
3138                        current->pid);
3139                error = sep_free_dcb_handler(sep, dma_ctx);
3140                dev_dbg(&sep->pdev->dev, "[PID%d] SEP_IOCFREEDCB end\n",
3141                        current->pid);
3142                break;
3143        default:
3144                error = -ENOTTY;
3145                dev_dbg(&sep->pdev->dev, "[PID%d] default end\n",
3146                        current->pid);
3147                break;
3148        }
3149
3150end_function:
3151        dev_dbg(&sep->pdev->dev, "[PID%d] ioctl end\n", current->pid);
3152
3153        return error;
3154}
3155
3156/**
3157 * sep_inthandler - interrupt handler for sep device
3158 * @irq: interrupt
3159 * @dev_id: device id
3160 */
3161static irqreturn_t sep_inthandler(int irq, void *dev_id)
3162{
3163        unsigned long lock_irq_flag;
3164        u32 reg_val, reg_val2 = 0;
3165        struct sep_device *sep = dev_id;
3166        irqreturn_t int_error = IRQ_HANDLED;
3167
3168        /* Are we in power save? */
3169#if defined(CONFIG_PM_RUNTIME) && defined(SEP_ENABLE_RUNTIME_PM)
3170        if (sep->pdev->dev.power.runtime_status != RPM_ACTIVE) {
3171                dev_dbg(&sep->pdev->dev, "interrupt during pwr save\n");
3172                return IRQ_NONE;
3173        }
3174#endif
3175
3176        if (test_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags) == 0) {
3177                dev_dbg(&sep->pdev->dev, "interrupt while nobody using sep\n");
3178                return IRQ_NONE;
3179        }
3180
3181        /* Read the IRR register to check if this is SEP interrupt */
3182        reg_val = sep_read_reg(sep, HW_HOST_IRR_REG_ADDR);
3183
3184        dev_dbg(&sep->pdev->dev, "sep int: IRR REG val: %x\n", reg_val);
3185
3186        if (reg_val & (0x1 << 13)) {
3187
3188                /* Lock and update the counter of reply messages */
3189                spin_lock_irqsave(&sep->snd_rply_lck, lock_irq_flag);
3190                sep->reply_ct++;
3191                spin_unlock_irqrestore(&sep->snd_rply_lck, lock_irq_flag);
3192
3193                dev_dbg(&sep->pdev->dev, "sep int: send_ct %lx reply_ct %lx\n",
3194                                        sep->send_ct, sep->reply_ct);
3195
3196                /* Is this a kernel client request */
3197                if (sep->in_kernel) {
3198                        tasklet_schedule(&sep->finish_tasklet);
3199                        goto finished_interrupt;
3200                }
3201
3202                /* Is this printf or daemon request? */
3203                reg_val2 = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR);
3204                dev_dbg(&sep->pdev->dev,
3205                        "SEP Interrupt - GPR2 is %08x\n", reg_val2);
3206
3207                clear_bit(SEP_WORKING_LOCK_BIT, &sep->in_use_flags);
3208
3209                if ((reg_val2 >> 30) & 0x1) {
3210                        dev_dbg(&sep->pdev->dev, "int: printf request\n");
3211                } else if (reg_val2 >> 31) {
3212                        dev_dbg(&sep->pdev->dev, "int: daemon request\n");
3213                } else {
3214                        dev_dbg(&sep->pdev->dev, "int: SEP reply\n");
3215                        wake_up(&sep->event_interrupt);
3216                }
3217        } else {
3218                dev_dbg(&sep->pdev->dev, "int: not SEP interrupt\n");
3219                int_error = IRQ_NONE;
3220        }
3221
3222finished_interrupt:
3223
3224        if (int_error == IRQ_HANDLED)
3225                sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, reg_val);
3226
3227        return int_error;
3228}
3229
3230/**
3231 * sep_reconfig_shared_area - reconfigure shared area
3232 * @sep: pointer to struct sep_device
3233 *
3234 * Reconfig the shared area between HOST and SEP - needed in case
3235 * the DX_CC_Init function was called before OS loading.
3236 */
3237static int sep_reconfig_shared_area(struct sep_device *sep)
3238{
3239        int ret_val;
3240
3241        /* use to limit waiting for SEP */
3242        unsigned long end_time;
3243
3244        /* Send the new SHARED MESSAGE AREA to the SEP */
3245        dev_dbg(&sep->pdev->dev, "reconfig shared; sending %08llx to sep\n",
3246                                (unsigned long long)sep->shared_bus);
3247
3248        sep_write_reg(sep, HW_HOST_HOST_SEP_GPR1_REG_ADDR, sep->shared_bus);
3249
3250        /* Poll for SEP response */
3251        ret_val = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR1_REG_ADDR);
3252
3253        end_time = jiffies + (WAIT_TIME * HZ);
3254
3255        while ((time_before(jiffies, end_time)) && (ret_val != 0xffffffff) &&
3256                (ret_val != sep->shared_bus))
3257                ret_val = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR1_REG_ADDR);
3258
3259        /* Check the return value (register) */
3260        if (ret_val != sep->shared_bus) {
3261                dev_warn(&sep->pdev->dev, "could not reconfig shared area\n");
3262                dev_warn(&sep->pdev->dev, "result was %x\n", ret_val);
3263                ret_val = -ENOMEM;
3264        } else
3265                ret_val = 0;
3266
3267        dev_dbg(&sep->pdev->dev, "reconfig shared area end\n");
3268
3269        return ret_val;
3270}
3271
3272/**
3273 *      sep_activate_dcb_dmatables_context - Takes DCB & DMA tables
3274 *                                              contexts into use
3275 *      @sep: SEP device
3276 *      @dcb_region: DCB region copy
3277 *      @dmatables_region: MLLI/DMA tables copy
3278 *      @dma_ctx: DMA context for current transaction
3279 */
3280ssize_t sep_activate_dcb_dmatables_context(struct sep_device *sep,
3281                                        struct sep_dcblock **dcb_region,
3282                                        void **dmatables_region,
3283                                        struct sep_dma_context *dma_ctx)
3284{
3285        void *dmaregion_free_start = NULL;
3286        void *dmaregion_free_end = NULL;
3287        void *dcbregion_free_start = NULL;
3288        void *dcbregion_free_end = NULL;
3289        ssize_t error = 0;
3290
3291        dev_dbg(&sep->pdev->dev, "[PID%d] activating dcb/dma region\n",
3292                current->pid);
3293
3294        if (1 > dma_ctx->nr_dcb_creat) {
3295                dev_warn(&sep->pdev->dev,
3296                         "[PID%d] invalid number of dcbs to activate 0x%08X\n",
3297                         current->pid, dma_ctx->nr_dcb_creat);
3298                error = -EINVAL;
3299                goto end_function;
3300        }
3301
3302        dmaregion_free_start = sep->shared_addr
3303                                + SYNCHRONIC_DMA_TABLES_AREA_OFFSET_BYTES;
3304        dmaregion_free_end = dmaregion_free_start
3305                                + SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES - 1;
3306
3307        if (dmaregion_free_start
3308             + dma_ctx->dmatables_len > dmaregion_free_end) {
3309                error = -ENOMEM;
3310                goto end_function;
3311        }
3312        memcpy(dmaregion_free_start,
3313               *dmatables_region,
3314               dma_ctx->dmatables_len);
3315        /* Free MLLI table copy */
3316        kfree(*dmatables_region);
3317        *dmatables_region = NULL;
3318
3319        /* Copy thread's DCB  table copy to DCB table region */
3320        dcbregion_free_start = sep->shared_addr +
3321                                SEP_DRIVER_SYSTEM_DCB_MEMORY_OFFSET_IN_BYTES;
3322        dcbregion_free_end = dcbregion_free_start +
3323                                (SEP_MAX_NUM_SYNC_DMA_OPS *
3324                                        sizeof(struct sep_dcblock)) - 1;
3325
3326        if (dcbregion_free_start
3327             + (dma_ctx->nr_dcb_creat * sizeof(struct sep_dcblock))
3328             > dcbregion_free_end) {
3329                error = -ENOMEM;
3330                goto end_function;
3331        }
3332
3333        memcpy(dcbregion_free_start,
3334               *dcb_region,
3335               dma_ctx->nr_dcb_creat * sizeof(struct sep_dcblock));
3336
3337        /* Print the tables */
3338        dev_dbg(&sep->pdev->dev, "activate: input table\n");
3339        sep_debug_print_lli_tables(sep,
3340                (struct sep_lli_entry *)sep_shared_area_bus_to_virt(sep,
3341                (*dcb_region)->input_mlli_address),
3342                (*dcb_region)->input_mlli_num_entries,
3343                (*dcb_region)->input_mlli_data_size);
3344
3345        dev_dbg(&sep->pdev->dev, "activate: output table\n");
3346        sep_debug_print_lli_tables(sep,
3347                (struct sep_lli_entry *)sep_shared_area_bus_to_virt(sep,
3348                (*dcb_region)->output_mlli_address),
3349                (*dcb_region)->output_mlli_num_entries,
3350                (*dcb_region)->output_mlli_data_size);
3351
3352        dev_dbg(&sep->pdev->dev,
3353                 "[PID%d] printing activated tables\n", current->pid);
3354
3355end_function:
3356        kfree(*dmatables_region);
3357        *dmatables_region = NULL;
3358
3359        kfree(*dcb_region);
3360        *dcb_region = NULL;
3361
3362        return error;
3363}
3364
3365/**
3366 *      sep_create_dcb_dmatables_context - Creates DCB & MLLI/DMA table context
3367 *      @sep: SEP device
3368 *      @dcb_region: DCB region buf to create for current transaction
3369 *      @dmatables_region: MLLI/DMA tables buf to create for current transaction
3370 *      @dma_ctx: DMA context buf to create for current transaction
3371 *      @user_dcb_args: User arguments for DCB/MLLI creation
3372 *      @num_dcbs: Number of DCBs to create
3373 *      @secure_dma: Indicate use of IMR restricted memory secure dma
3374 */
3375static ssize_t sep_create_dcb_dmatables_context(struct sep_device *sep,
3376                        struct sep_dcblock **dcb_region,
3377                        void **dmatables_region,
3378                        struct sep_dma_context **dma_ctx,
3379                        const struct build_dcb_struct __user *user_dcb_args,
3380                        const u32 num_dcbs, bool secure_dma)
3381{
3382        int error = 0;
3383        int i = 0;
3384        struct build_dcb_struct *dcb_args = NULL;
3385
3386        dev_dbg(&sep->pdev->dev, "[PID%d] creating dcb/dma region\n",
3387                current->pid);
3388
3389        if (!dcb_region || !dma_ctx || !dmatables_region || !user_dcb_args) {
3390                error = -EINVAL;
3391                goto end_function;
3392        }
3393
3394        if (SEP_MAX_NUM_SYNC_DMA_OPS < num_dcbs) {
3395                dev_warn(&sep->pdev->dev,
3396                         "[PID%d] invalid number of dcbs 0x%08X\n",
3397                         current->pid, num_dcbs);
3398                error = -EINVAL;
3399                goto end_function;
3400        }
3401
3402        dcb_args = kcalloc(num_dcbs, sizeof(struct build_dcb_struct),
3403                           GFP_KERNEL);
3404        if (!dcb_args) {
3405                error = -ENOMEM;
3406                goto end_function;
3407        }
3408
3409        if (copy_from_user(dcb_args,
3410                        user_dcb_args,
3411                        num_dcbs * sizeof(struct build_dcb_struct))) {
3412                error = -EFAULT;
3413                goto end_function;
3414        }
3415
3416        /* Allocate thread-specific memory for DCB */
3417        *dcb_region = kzalloc(num_dcbs * sizeof(struct sep_dcblock),
3418                              GFP_KERNEL);
3419        if (!(*dcb_region)) {
3420                error = -ENOMEM;
3421                goto end_function;
3422        }
3423
3424        /* Prepare DCB and MLLI table into the allocated regions */
3425        for (i = 0; i < num_dcbs; i++) {
3426                error = sep_prepare_input_output_dma_table_in_dcb(sep,
3427                                (unsigned long)dcb_args[i].app_in_address,
3428                                (unsigned long)dcb_args[i].app_out_address,
3429                                dcb_args[i].data_in_size,
3430                                dcb_args[i].block_size,
3431                                dcb_args[i].tail_block_size,
3432                                dcb_args[i].is_applet,
3433                                false, secure_dma,
3434                                *dcb_region, dmatables_region,
3435                                dma_ctx,
3436                                NULL,
3437                                NULL);
3438                if (error) {
3439                        dev_warn(&sep->pdev->dev,
3440                                 "[PID%d] dma table creation failed\n",
3441                                 current->pid);
3442                        goto end_function;
3443                }
3444
3445                if (dcb_args[i].app_in_address != 0)
3446                        (*dma_ctx)->input_data_len += dcb_args[i].data_in_size;
3447        }
3448
3449end_function:
3450        kfree(dcb_args);
3451        return error;
3452
3453}
3454
3455/**
3456 *      sep_create_dcb_dmatables_context_kernel - Creates DCB & MLLI/DMA table context
3457 *      for kernel crypto
3458 *      @sep: SEP device
3459 *      @dcb_region: DCB region buf to create for current transaction
3460 *      @dmatables_region: MLLI/DMA tables buf to create for current transaction
3461 *      @dma_ctx: DMA context buf to create for current transaction
3462 *      @user_dcb_args: User arguments for DCB/MLLI creation
3463 *      @num_dcbs: Number of DCBs to create
3464 *      This does that same thing as sep_create_dcb_dmatables_context
3465 *      except that it is used only for the kernel crypto operation. It is
3466 *      separate because there is no user data involved; the dcb data structure
3467 *      is specific for kernel crypto (build_dcb_struct_kernel)
3468 */
3469int sep_create_dcb_dmatables_context_kernel(struct sep_device *sep,
3470                        struct sep_dcblock **dcb_region,
3471                        void **dmatables_region,
3472                        struct sep_dma_context **dma_ctx,
3473                        const struct build_dcb_struct_kernel *dcb_data,
3474                        const u32 num_dcbs)
3475{
3476        int error = 0;
3477        int i = 0;
3478
3479        dev_dbg(&sep->pdev->dev, "[PID%d] creating dcb/dma region\n",
3480                current->pid);
3481
3482        if (!dcb_region || !dma_ctx || !dmatables_region || !dcb_data) {
3483                error = -EINVAL;
3484                goto end_function;
3485        }
3486
3487        if (SEP_MAX_NUM_SYNC_DMA_OPS < num_dcbs) {
3488                dev_warn(&sep->pdev->dev,
3489                         "[PID%d] invalid number of dcbs 0x%08X\n",
3490                         current->pid, num_dcbs);
3491                error = -EINVAL;
3492                goto end_function;
3493        }
3494
3495        dev_dbg(&sep->pdev->dev, "[PID%d] num_dcbs is %d\n",
3496                current->pid, num_dcbs);
3497
3498        /* Allocate thread-specific memory for DCB */
3499        *dcb_region = kzalloc(num_dcbs * sizeof(struct sep_dcblock),
3500                              GFP_KERNEL);
3501        if (!(*dcb_region)) {
3502                error = -ENOMEM;
3503                goto end_function;
3504        }
3505
3506        /* Prepare DCB and MLLI table into the allocated regions */
3507        for (i = 0; i < num_dcbs; i++) {
3508                error = sep_prepare_input_output_dma_table_in_dcb(sep,
3509                                (unsigned long)dcb_data->app_in_address,
3510                                (unsigned long)dcb_data->app_out_address,
3511                                dcb_data->data_in_size,
3512                                dcb_data->block_size,
3513                                dcb_data->tail_block_size,
3514                                dcb_data->is_applet,
3515                                true,
3516                                false,
3517                                *dcb_region, dmatables_region,
3518                                dma_ctx,
3519                                dcb_data->src_sg,
3520                                dcb_data->dst_sg);
3521                if (error) {
3522                        dev_warn(&sep->pdev->dev,
3523                                 "[PID%d] dma table creation failed\n",
3524                                 current->pid);
3525                        goto end_function;
3526                }
3527        }
3528
3529end_function:
3530        return error;
3531
3532}
3533
3534/**
3535 *      sep_activate_msgarea_context - Takes the message area context into use
3536 *      @sep: SEP device
3537 *      @msg_region: Message area context buf
3538 *      @msg_len: Message area context buffer size
3539 */
3540static ssize_t sep_activate_msgarea_context(struct sep_device *sep,
3541                                            void **msg_region,
3542                                            const size_t msg_len)
3543{
3544        dev_dbg(&sep->pdev->dev, "[PID%d] activating msg region\n",
3545                current->pid);
3546
3547        if (!msg_region || !(*msg_region) ||
3548            SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES < msg_len) {
3549                dev_warn(&sep->pdev->dev,
3550                         "[PID%d] invalid act msgarea len 0x%08zX\n",
3551                         current->pid, msg_len);
3552                return -EINVAL;
3553        }
3554
3555        memcpy(sep->shared_addr, *msg_region, msg_len);
3556
3557        return 0;
3558}
3559
3560/**
3561 *      sep_create_msgarea_context - Creates message area context
3562 *      @sep: SEP device
3563 *      @msg_region: Msg area region buf to create for current transaction
3564 *      @msg_user: Content for msg area region from user
3565 *      @msg_len: Message area size
3566 */
3567static ssize_t sep_create_msgarea_context(struct sep_device *sep,
3568                                          void **msg_region,
3569                                          const void __user *msg_user,
3570                                          const size_t msg_len)
3571{
3572        int error = 0;
3573
3574        dev_dbg(&sep->pdev->dev, "[PID%d] creating msg region\n",
3575                current->pid);
3576
3577        if (!msg_region ||
3578            !msg_user ||
3579            SEP_DRIVER_MAX_MESSAGE_SIZE_IN_BYTES < msg_len ||
3580            SEP_DRIVER_MIN_MESSAGE_SIZE_IN_BYTES > msg_len) {
3581                dev_warn(&sep->pdev->dev,
3582                         "[PID%d] invalid creat msgarea len 0x%08zX\n",
3583                         current->pid, msg_len);
3584                error = -EINVAL;
3585                goto end_function;
3586        }
3587
3588        /* Allocate thread-specific memory for message buffer */
3589        *msg_region = kzalloc(msg_len, GFP_KERNEL);
3590        if (!(*msg_region)) {
3591                error = -ENOMEM;
3592                goto end_function;
3593        }
3594
3595        /* Copy input data to write() to allocated message buffer */
3596        if (copy_from_user(*msg_region, msg_user, msg_len)) {
3597                error = -EFAULT;
3598                goto end_function;
3599        }
3600
3601end_function:
3602        if (error && msg_region) {
3603                kfree(*msg_region);
3604                *msg_region = NULL;
3605        }
3606
3607        return error;
3608}
3609
3610
3611/**
3612 *      sep_read - Returns results of an operation for fastcall interface
3613 *      @filp: File pointer
3614 *      @buf_user: User buffer for storing results
3615 *      @count_user: User buffer size
3616 *      @offset: File offset, not supported
3617 *
3618 *      The implementation does not support reading in chunks, all data must be
3619 *      consumed during a single read system call.
3620 */
3621static ssize_t sep_read(struct file *filp,
3622                        char __user *buf_user, size_t count_user,
3623                        loff_t *offset)
3624{
3625        struct sep_private_data * const private_data = filp->private_data;
3626        struct sep_call_status *call_status = &private_data->call_status;
3627        struct sep_device *sep = private_data->device;
3628        struct sep_dma_context **dma_ctx = &private_data->dma_ctx;
3629        struct sep_queue_info **my_queue_elem = &private_data->my_queue_elem;
3630        ssize_t error = 0, error_tmp = 0;
3631
3632        /* Am I the process that owns the transaction? */
3633        error = sep_check_transaction_owner(sep);
3634        if (error) {
3635                dev_dbg(&sep->pdev->dev, "[PID%d] read pid is not owner\n",
3636                        current->pid);
3637                goto end_function;
3638        }
3639
3640        /* Checks that user has called necessary apis */
3641        if (0 == test_bit(SEP_FASTCALL_WRITE_DONE_OFFSET,
3642                        &call_status->status)) {
3643                dev_warn(&sep->pdev->dev,
3644                         "[PID%d] fastcall write not called\n",
3645                         current->pid);
3646                error = -EPROTO;
3647                goto end_function_error;
3648        }
3649
3650        if (!buf_user) {
3651                dev_warn(&sep->pdev->dev,
3652                         "[PID%d] null user buffer\n",
3653                         current->pid);
3654                error = -EINVAL;
3655                goto end_function_error;
3656        }
3657
3658
3659        /* Wait for SEP to finish */
3660        wait_event(sep->event_interrupt,
3661                   test_bit(SEP_WORKING_LOCK_BIT,
3662                            &sep->in_use_flags) == 0);
3663
3664        sep_dump_message(sep);
3665
3666        dev_dbg(&sep->pdev->dev, "[PID%d] count_user = 0x%08zX\n",
3667                current->pid, count_user);
3668
3669        /* In case user has allocated bigger buffer */
3670        if (count_user > SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES)
3671                count_user = SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES;
3672
3673        if (copy_to_user(buf_user, sep->shared_addr, count_user)) {
3674                error = -EFAULT;
3675                goto end_function_error;
3676        }
3677
3678        dev_dbg(&sep->pdev->dev, "[PID%d] read succeeded\n", current->pid);
3679        error = count_user;
3680
3681end_function_error:
3682        /* Copy possible tail data to user and free DCB and MLLIs */
3683        error_tmp = sep_free_dcb_handler(sep, dma_ctx);
3684        if (error_tmp)
3685                dev_warn(&sep->pdev->dev, "[PID%d] dcb free failed\n",
3686                        current->pid);
3687
3688        /* End the transaction, wakeup pending ones */
3689        error_tmp = sep_end_transaction_handler(sep, dma_ctx, call_status,
3690                my_queue_elem);
3691        if (error_tmp)
3692                dev_warn(&sep->pdev->dev,
3693                         "[PID%d] ending transaction failed\n",
3694                         current->pid);
3695
3696end_function:
3697        return error;
3698}
3699
3700/**
3701 *      sep_fastcall_args_get - Gets fastcall params from user
3702 *      sep: SEP device
3703 *      @args: Parameters buffer
3704 *      @buf_user: User buffer for operation parameters
3705 *      @count_user: User buffer size
3706 */
3707static inline ssize_t sep_fastcall_args_get(struct sep_device *sep,
3708                                            struct sep_fastcall_hdr *args,
3709                                            const char __user *buf_user,
3710                                            const size_t count_user)
3711{
3712        ssize_t error = 0;
3713        size_t actual_count = 0;
3714
3715        if (!buf_user) {
3716                dev_warn(&sep->pdev->dev,
3717                         "[PID%d] null user buffer\n",
3718                         current->pid);
3719                error = -EINVAL;
3720                goto end_function;
3721        }
3722
3723        if (count_user < sizeof(struct sep_fastcall_hdr)) {
3724                dev_warn(&sep->pdev->dev,
3725                         "[PID%d] too small message size 0x%08zX\n",
3726                         current->pid, count_user);
3727                error = -EINVAL;
3728                goto end_function;
3729        }
3730
3731
3732        if (copy_from_user(args, buf_user, sizeof(struct sep_fastcall_hdr))) {
3733                error = -EFAULT;
3734                goto end_function;
3735        }
3736
3737        if (SEP_FC_MAGIC != args->magic) {
3738                dev_warn(&sep->pdev->dev,
3739                         "[PID%d] invalid fastcall magic 0x%08X\n",
3740                         current->pid, args->magic);
3741                error = -EINVAL;
3742                goto end_function;
3743        }
3744
3745        dev_dbg(&sep->pdev->dev, "[PID%d] fastcall hdr num of DCBs 0x%08X\n",
3746                current->pid, args->num_dcbs);
3747        dev_dbg(&sep->pdev->dev, "[PID%d] fastcall hdr msg len 0x%08X\n",
3748                current->pid, args->msg_len);
3749
3750        if (SEP_DRIVER_MAX_MESSAGE_SIZE_IN_BYTES < args->msg_len ||
3751            SEP_DRIVER_MIN_MESSAGE_SIZE_IN_BYTES > args->msg_len) {
3752                dev_warn(&sep->pdev->dev,
3753                         "[PID%d] invalid message length\n",
3754                         current->pid);
3755                error = -EINVAL;
3756                goto end_function;
3757        }
3758
3759        actual_count = sizeof(struct sep_fastcall_hdr)
3760                        + args->msg_len
3761                        + (args->num_dcbs * sizeof(struct build_dcb_struct));
3762
3763        if (actual_count != count_user) {
3764                dev_warn(&sep->pdev->dev,
3765                         "[PID%d] inconsistent message "
3766                         "sizes 0x%08zX vs 0x%08zX\n",
3767                         current->pid, actual_count, count_user);
3768                error = -EMSGSIZE;
3769                goto end_function;
3770        }
3771
3772end_function:
3773        return error;
3774}
3775
3776/**
3777 *      sep_write - Starts an operation for fastcall interface
3778 *      @filp: File pointer
3779 *      @buf_user: User buffer for operation parameters
3780 *      @count_user: User buffer size
3781 *      @offset: File offset, not supported
3782 *
3783 *      The implementation does not support writing in chunks,
3784 *      all data must be given during a single write system call.
3785 */
3786static ssize_t sep_write(struct file *filp,
3787                         const char __user *buf_user, size_t count_user,
3788                         loff_t *offset)
3789{
3790        struct sep_private_data * const private_data = filp->private_data;
3791        struct sep_call_status *call_status = &private_data->call_status;
3792        struct sep_device *sep = private_data->device;
3793        struct sep_dma_context *dma_ctx = NULL;
3794        struct sep_fastcall_hdr call_hdr = {0};
3795        void *msg_region = NULL;
3796        void *dmatables_region = NULL;
3797        struct sep_dcblock *dcb_region = NULL;
3798        ssize_t error = 0;
3799        struct sep_queue_info *my_queue_elem = NULL;
3800        bool my_secure_dma; /* are we using secure_dma (IMR)? */
3801
3802        dev_dbg(&sep->pdev->dev, "[PID%d] sep dev is 0x%p\n",
3803                current->pid, sep);
3804        dev_dbg(&sep->pdev->dev, "[PID%d] private_data is 0x%p\n",
3805                current->pid, private_data);
3806
3807        error = sep_fastcall_args_get(sep, &call_hdr, buf_user, count_user);
3808        if (error)
3809                goto end_function;
3810
3811        buf_user += sizeof(struct sep_fastcall_hdr);
3812
3813        if (call_hdr.secure_dma == 0)
3814                my_secure_dma = false;
3815        else
3816                my_secure_dma = true;
3817
3818        /*
3819         * Controlling driver memory usage by limiting amount of
3820         * buffers created. Only SEP_DOUBLEBUF_USERS_LIMIT number
3821         * of threads can progress further at a time
3822         */
3823        dev_dbg(&sep->pdev->dev,
3824                "[PID%d] waiting for double buffering region access\n",
3825                current->pid);
3826        error = down_interruptible(&sep->sep_doublebuf);
3827        dev_dbg(&sep->pdev->dev, "[PID%d] double buffering region start\n",
3828                                        current->pid);
3829        if (error) {
3830                /* Signal received */
3831                goto end_function_error;
3832        }
3833
3834
3835        /*
3836         * Prepare contents of the shared area regions for
3837         * the operation into temporary buffers
3838         */
3839        if (0 < call_hdr.num_dcbs) {
3840                error = sep_create_dcb_dmatables_context(sep,
3841                                &dcb_region,
3842                                &dmatables_region,
3843                                &dma_ctx,
3844                                (const struct build_dcb_struct __user *)
3845                                        buf_user,
3846                                call_hdr.num_dcbs, my_secure_dma);
3847                if (error)
3848                        goto end_function_error_doublebuf;
3849
3850                buf_user += call_hdr.num_dcbs * sizeof(struct build_dcb_struct);
3851        }
3852
3853        error = sep_create_msgarea_context(sep,
3854                                           &msg_region,
3855                                           buf_user,
3856                                           call_hdr.msg_len);
3857        if (error)
3858                goto end_function_error_doublebuf;
3859
3860        dev_dbg(&sep->pdev->dev, "[PID%d] updating queue status\n",
3861                                                        current->pid);
3862        my_queue_elem = sep_queue_status_add(sep,
3863                                ((struct sep_msgarea_hdr *)msg_region)->opcode,
3864                                (dma_ctx) ? dma_ctx->input_data_len : 0,
3865                                     current->pid,
3866                                     current->comm, sizeof(current->comm));
3867
3868        if (!my_queue_elem) {
3869                dev_dbg(&sep->pdev->dev,
3870                        "[PID%d] updating queue status error\n", current->pid);
3871                error = -ENOMEM;
3872                goto end_function_error_doublebuf;
3873        }
3874
3875        /* Wait until current process gets the transaction */
3876        error = sep_wait_transaction(sep);
3877
3878        if (error) {
3879                /* Interrupted by signal, don't clear transaction */
3880                dev_dbg(&sep->pdev->dev, "[PID%d] interrupted by signal\n",
3881                        current->pid);
3882                sep_queue_status_remove(sep, &my_queue_elem);
3883                goto end_function_error_doublebuf;
3884        }
3885
3886        dev_dbg(&sep->pdev->dev, "[PID%d] saving queue element\n",
3887                current->pid);
3888        private_data->my_queue_elem = my_queue_elem;
3889
3890        /* Activate shared area regions for the transaction */
3891        error = sep_activate_msgarea_context(sep, &msg_region,
3892                                             call_hdr.msg_len);
3893        if (error)
3894                goto end_function_error_clear_transact;
3895
3896        sep_dump_message(sep);
3897
3898        if (0 < call_hdr.num_dcbs) {
3899                error = sep_activate_dcb_dmatables_context(sep,
3900                                &dcb_region,
3901                                &dmatables_region,
3902                                dma_ctx);
3903                if (error)
3904                        goto end_function_error_clear_transact;
3905        }
3906
3907        /* Send command to SEP */
3908        error = sep_send_command_handler(sep);
3909        if (error)
3910                goto end_function_error_clear_transact;
3911
3912        /* Store DMA context for the transaction */
3913        private_data->dma_ctx = dma_ctx;
3914        /* Update call status */
3915        set_bit(SEP_FASTCALL_WRITE_DONE_OFFSET, &call_status->status);
3916        error = count_user;
3917
3918        up(&sep->sep_doublebuf);
3919        dev_dbg(&sep->pdev->dev, "[PID%d] double buffering region end\n",
3920                current->pid);
3921
3922        goto end_function;
3923
3924end_function_error_clear_transact:
3925        sep_end_transaction_handler(sep, &dma_ctx, call_status,
3926                                                &private_data->my_queue_elem);
3927
3928end_function_error_doublebuf:
3929        up(&sep->sep_doublebuf);
3930        dev_dbg(&sep->pdev->dev, "[PID%d] double buffering region end\n",
3931                current->pid);
3932
3933end_function_error:
3934        if (dma_ctx)
3935                sep_free_dma_table_data_handler(sep, &dma_ctx);
3936
3937end_function:
3938        kfree(dcb_region);
3939        kfree(dmatables_region);
3940        kfree(msg_region);
3941
3942        return error;
3943}
3944/**
3945 *      sep_seek - Handler for seek system call
3946 *      @filp: File pointer
3947 *      @offset: File offset
3948 *      @origin: Options for offset
3949 *
3950 *      Fastcall interface does not support seeking, all reads
3951 *      and writes are from/to offset zero
3952 */
3953static loff_t sep_seek(struct file *filp, loff_t offset, int origin)
3954{
3955        return -ENOSYS;
3956}
3957
3958
3959
3960/**
3961 * sep_file_operations - file operation on sep device
3962 * @sep_ioctl:  ioctl handler from user space call
3963 * @sep_poll:   poll handler
3964 * @sep_open:   handles sep device open request
3965 * @sep_release:handles sep device release request
3966 * @sep_mmap:   handles memory mapping requests
3967 * @sep_read:   handles read request on sep device
3968 * @sep_write:  handles write request on sep device
3969 * @sep_seek:   handles seek request on sep device
3970 */
3971static const struct file_operations sep_file_operations = {
3972        .owner = THIS_MODULE,
3973        .unlocked_ioctl = sep_ioctl,
3974        .poll = sep_poll,
3975        .open = sep_open,
3976        .release = sep_release,
3977        .mmap = sep_mmap,
3978        .read = sep_read,
3979        .write = sep_write,
3980        .llseek = sep_seek,
3981};
3982
3983/**
3984 * sep_sysfs_read - read sysfs entry per gives arguments
3985 * @filp: file pointer
3986 * @kobj: kobject pointer
3987 * @attr: binary file attributes
3988 * @buf: read to this buffer
3989 * @pos: offset to read
3990 * @count: amount of data to read
3991 *
3992 * This function is to read sysfs entries for sep driver per given arguments.
3993 */
3994static ssize_t
3995sep_sysfs_read(struct file *filp, struct kobject *kobj,
3996                struct bin_attribute *attr,
3997                char *buf, loff_t pos, size_t count)
3998{
3999        unsigned long lck_flags;
4000        size_t nleft = count;
4001        struct sep_device *sep = sep_dev;
4002        struct sep_queue_info *queue_elem = NULL;
4003        u32 queue_num = 0;
4004        u32 i = 1;
4005
4006        spin_lock_irqsave(&sep->sep_queue_lock, lck_flags);
4007
4008        queue_num = sep->sep_queue_num;
4009        if (queue_num > SEP_DOUBLEBUF_USERS_LIMIT)
4010                queue_num = SEP_DOUBLEBUF_USERS_LIMIT;
4011
4012
4013        if (count < sizeof(queue_num)
4014                        + (queue_num * sizeof(struct sep_queue_data))) {
4015                spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags);
4016                return -EINVAL;
4017        }
4018
4019        memcpy(buf, &queue_num, sizeof(queue_num));
4020        buf += sizeof(queue_num);
4021        nleft -= sizeof(queue_num);
4022
4023        list_for_each_entry(queue_elem, &sep->sep_queue_status, list) {
4024                if (i++ > queue_num)
4025                        break;
4026
4027                memcpy(buf, &queue_elem->data, sizeof(queue_elem->data));
4028                nleft -= sizeof(queue_elem->data);
4029                buf += sizeof(queue_elem->data);
4030        }
4031        spin_unlock_irqrestore(&sep->sep_queue_lock, lck_flags);
4032
4033        return count - nleft;
4034}
4035
4036/**
4037 * bin_attributes - defines attributes for queue_status
4038 * @attr: attributes (name & permissions)
4039 * @read: function pointer to read this file
4040 * @size: maxinum size of binary attribute
4041 */
4042static const struct bin_attribute queue_status = {
4043        .attr = {.name = "queue_status", .mode = 0444},
4044        .read = sep_sysfs_read,
4045        .size = sizeof(u32)
4046                + (SEP_DOUBLEBUF_USERS_LIMIT * sizeof(struct sep_queue_data)),
4047};
4048
4049/**
4050 * sep_register_driver_with_fs - register misc devices
4051 * @sep: pointer to struct sep_device
4052 *
4053 * This function registers the driver with the file system
4054 */
4055static int sep_register_driver_with_fs(struct sep_device *sep)
4056{
4057        int ret_val;
4058
4059        sep->miscdev_sep.minor = MISC_DYNAMIC_MINOR;
4060        sep->miscdev_sep.name = SEP_DEV_NAME;
4061        sep->miscdev_sep.fops = &sep_file_operations;
4062
4063        ret_val = misc_register(&sep->miscdev_sep);
4064        if (ret_val) {
4065                dev_warn(&sep->pdev->dev, "misc reg fails for SEP %x\n",
4066                        ret_val);
4067                return ret_val;
4068        }
4069
4070        ret_val = device_create_bin_file(sep->miscdev_sep.this_device,
4071                                                                &queue_status);
4072        if (ret_val) {
4073                dev_warn(&sep->pdev->dev, "sysfs attribute1 fails for SEP %x\n",
4074                        ret_val);
4075                misc_deregister(&sep->miscdev_sep);
4076                return ret_val;
4077        }
4078
4079        return ret_val;
4080}
4081
4082
4083/**
4084 *sep_probe - probe a matching PCI device
4085 *@pdev:        pci_device
4086 *@ent: pci_device_id
4087 *
4088 *Attempt to set up and configure a SEP device that has been
4089 *discovered by the PCI layer. Allocates all required resources.
4090 */
4091static int sep_probe(struct pci_dev *pdev,
4092        const struct pci_device_id *ent)
4093{
4094        int error = 0;
4095        struct sep_device *sep = NULL;
4096
4097        if (sep_dev != NULL) {
4098                dev_dbg(&pdev->dev, "only one SEP supported.\n");
4099                return -EBUSY;
4100        }
4101
4102        /* Enable the device */
4103        error = pci_enable_device(pdev);
4104        if (error) {
4105                dev_warn(&pdev->dev, "error enabling pci device\n");
4106                goto end_function;
4107        }
4108
4109        /* Allocate the sep_device structure for this device */
4110        sep_dev = kzalloc(sizeof(struct sep_device), GFP_ATOMIC);
4111        if (sep_dev == NULL) {
4112                error = -ENOMEM;
4113                goto end_function_disable_device;
4114        }
4115
4116        /*
4117         * We're going to use another variable for actually
4118         * working with the device; this way, if we have
4119         * multiple devices in the future, it would be easier
4120         * to make appropriate changes
4121         */
4122        sep = sep_dev;
4123
4124        sep->pdev = pci_dev_get(pdev);
4125
4126        init_waitqueue_head(&sep->event_transactions);
4127        init_waitqueue_head(&sep->event_interrupt);
4128        spin_lock_init(&sep->snd_rply_lck);
4129        spin_lock_init(&sep->sep_queue_lock);
4130        sema_init(&sep->sep_doublebuf, SEP_DOUBLEBUF_USERS_LIMIT);
4131
4132        INIT_LIST_HEAD(&sep->sep_queue_status);
4133
4134        dev_dbg(&sep->pdev->dev,
4135                "sep probe: PCI obtained, device being prepared\n");
4136
4137        /* Set up our register area */
4138        sep->reg_physical_addr = pci_resource_start(sep->pdev, 0);
4139        if (!sep->reg_physical_addr) {
4140                dev_warn(&sep->pdev->dev, "Error getting register start\n");
4141                error = -ENODEV;
4142                goto end_function_free_sep_dev;
4143        }
4144
4145        sep->reg_physical_end = pci_resource_end(sep->pdev, 0);
4146        if (!sep->reg_physical_end) {
4147                dev_warn(&sep->pdev->dev, "Error getting register end\n");
4148                error = -ENODEV;
4149                goto end_function_free_sep_dev;
4150        }
4151
4152        sep->reg_addr = ioremap_nocache(sep->reg_physical_addr,
4153                (size_t)(sep->reg_physical_end - sep->reg_physical_addr + 1));
4154        if (!sep->reg_addr) {
4155                dev_warn(&sep->pdev->dev, "Error getting register virtual\n");
4156                error = -ENODEV;
4157                goto end_function_free_sep_dev;
4158        }
4159
4160        dev_dbg(&sep->pdev->dev,
4161                "Register area start %llx end %llx virtual %p\n",
4162                (unsigned long long)sep->reg_physical_addr,
4163                (unsigned long long)sep->reg_physical_end,
4164                sep->reg_addr);
4165
4166        /* Allocate the shared area */
4167        sep->shared_size = SEP_DRIVER_MESSAGE_SHARED_AREA_SIZE_IN_BYTES +
4168                SYNCHRONIC_DMA_TABLES_AREA_SIZE_BYTES +
4169                SEP_DRIVER_DATA_POOL_SHARED_AREA_SIZE_IN_BYTES +
4170                SEP_DRIVER_STATIC_AREA_SIZE_IN_BYTES +
4171                SEP_DRIVER_SYSTEM_DATA_MEMORY_SIZE_IN_BYTES;
4172
4173        if (sep_map_and_alloc_shared_area(sep)) {
4174                error = -ENOMEM;
4175                /* Allocation failed */
4176                goto end_function_error;
4177        }
4178
4179        /* Clear ICR register */
4180        sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF);
4181
4182        /* Set the IMR register - open only GPR 2 */
4183        sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, (~(0x1 << 13)));
4184
4185        /* Read send/receive counters from SEP */
4186        sep->reply_ct = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR);
4187        sep->reply_ct &= 0x3FFFFFFF;
4188        sep->send_ct = sep->reply_ct;
4189
4190        /* Get the interrupt line */
4191        error = request_irq(pdev->irq, sep_inthandler, IRQF_SHARED,
4192                "sep_driver", sep);
4193
4194        if (error)
4195                goto end_function_deallocate_sep_shared_area;
4196
4197        /* The new chip requires a shared area reconfigure */
4198        error = sep_reconfig_shared_area(sep);
4199        if (error)
4200                goto end_function_free_irq;
4201
4202        sep->in_use = 1;
4203
4204        /* Finally magic up the device nodes */
4205        /* Register driver with the fs */
4206        error = sep_register_driver_with_fs(sep);
4207
4208        if (error) {
4209                dev_err(&sep->pdev->dev, "error registering dev file\n");
4210                goto end_function_free_irq;
4211        }
4212
4213        sep->in_use = 0; /* through touching the device */
4214#ifdef SEP_ENABLE_RUNTIME_PM
4215        pm_runtime_put_noidle(&sep->pdev->dev);
4216        pm_runtime_allow(&sep->pdev->dev);
4217        pm_runtime_set_autosuspend_delay(&sep->pdev->dev,
4218                SUSPEND_DELAY);
4219        pm_runtime_use_autosuspend(&sep->pdev->dev);
4220        pm_runtime_mark_last_busy(&sep->pdev->dev);
4221        sep->power_save_setup = 1;
4222#endif
4223        /* register kernel crypto driver */
4224#if defined(CONFIG_CRYPTO) || defined(CONFIG_CRYPTO_MODULE)
4225        error = sep_crypto_setup();
4226        if (error) {
4227                dev_err(&sep->pdev->dev, "crypto setup failed\n");
4228                goto end_function_free_irq;
4229        }
4230#endif
4231        goto end_function;
4232
4233end_function_free_irq:
4234        free_irq(pdev->irq, sep);
4235
4236end_function_deallocate_sep_shared_area:
4237        /* De-allocate shared area */
4238        sep_unmap_and_free_shared_area(sep);
4239
4240end_function_error:
4241        iounmap(sep->reg_addr);
4242
4243end_function_free_sep_dev:
4244        pci_dev_put(sep_dev->pdev);
4245        kfree(sep_dev);
4246        sep_dev = NULL;
4247
4248end_function_disable_device:
4249        pci_disable_device(pdev);
4250
4251end_function:
4252        return error;
4253}
4254
4255/**
4256 * sep_remove - handles removing device from pci subsystem
4257 * @pdev:       pointer to pci device
4258 *
4259 * This function will handle removing our sep device from pci subsystem on exit
4260 * or unloading this module. It should free up all used resources, and unmap if
4261 * any memory regions mapped.
4262 */
4263static void sep_remove(struct pci_dev *pdev)
4264{
4265        struct sep_device *sep = sep_dev;
4266
4267        /* Unregister from fs */
4268        misc_deregister(&sep->miscdev_sep);
4269
4270        /* Unregister from kernel crypto */
4271#if defined(CONFIG_CRYPTO) || defined(CONFIG_CRYPTO_MODULE)
4272        sep_crypto_takedown();
4273#endif
4274        /* Free the irq */
4275        free_irq(sep->pdev->irq, sep);
4276
4277        /* Free the shared area  */
4278        sep_unmap_and_free_shared_area(sep_dev);
4279        iounmap(sep_dev->reg_addr);
4280
4281#ifdef SEP_ENABLE_RUNTIME_PM
4282        if (sep->in_use) {
4283                sep->in_use = 0;
4284                pm_runtime_forbid(&sep->pdev->dev);
4285                pm_runtime_get_noresume(&sep->pdev->dev);
4286        }
4287#endif
4288        pci_dev_put(sep_dev->pdev);
4289        kfree(sep_dev);
4290        sep_dev = NULL;
4291}
4292
4293/* Initialize struct pci_device_id for our driver */
4294static const struct pci_device_id sep_pci_id_tbl[] = {
4295        {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0826)},
4296        {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08e9)},
4297        {0}
4298};
4299
4300/* Export our pci_device_id structure to user space */
4301MODULE_DEVICE_TABLE(pci, sep_pci_id_tbl);
4302
4303#ifdef SEP_ENABLE_RUNTIME_PM
4304
4305/**
4306 * sep_pm_resume - rsume routine while waking up from S3 state
4307 * @dev:        pointer to sep device
4308 *
4309 * This function is to be used to wake up sep driver while system awakes from S3
4310 * state i.e. suspend to ram. The RAM in intact.
4311 * Notes - revisit with more understanding of pm, ICR/IMR & counters.
4312 */
4313static int sep_pci_resume(struct device *dev)
4314{
4315        struct sep_device *sep = sep_dev;
4316
4317        dev_dbg(&sep->pdev->dev, "pci resume called\n");
4318
4319        if (sep->power_state == SEP_DRIVER_POWERON)
4320                return 0;
4321
4322        /* Clear ICR register */
4323        sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF);
4324
4325        /* Set the IMR register - open only GPR 2 */
4326        sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, (~(0x1 << 13)));
4327
4328        /* Read send/receive counters from SEP */
4329        sep->reply_ct = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR);
4330        sep->reply_ct &= 0x3FFFFFFF;
4331        sep->send_ct = sep->reply_ct;
4332
4333        sep->power_state = SEP_DRIVER_POWERON;
4334
4335        return 0;
4336}
4337
4338/**
4339 * sep_pm_suspend - suspend routine while going to S3 state
4340 * @dev:        pointer to sep device
4341 *
4342 * This function is to be used to suspend sep driver while system goes to S3
4343 * state i.e. suspend to ram. The RAM in intact and ON during this suspend.
4344 * Notes - revisit with more understanding of pm, ICR/IMR
4345 */
4346static int sep_pci_suspend(struct device *dev)
4347{
4348        struct sep_device *sep = sep_dev;
4349
4350        dev_dbg(&sep->pdev->dev, "pci suspend called\n");
4351        if (sep->in_use == 1)
4352                return -EAGAIN;
4353
4354        sep->power_state = SEP_DRIVER_POWEROFF;
4355
4356        /* Clear ICR register */
4357        sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF);
4358
4359        /* Set the IMR to block all */
4360        sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, 0xFFFFFFFF);
4361
4362        return 0;
4363}
4364
4365/**
4366 * sep_pm_runtime_resume - runtime resume routine
4367 * @dev:        pointer to sep device
4368 *
4369 * Notes - revisit with more understanding of pm, ICR/IMR & counters
4370 */
4371static int sep_pm_runtime_resume(struct device *dev)
4372{
4373
4374        u32 retval2;
4375        u32 delay_count;
4376        struct sep_device *sep = sep_dev;
4377
4378        dev_dbg(&sep->pdev->dev, "pm runtime resume called\n");
4379
4380        /**
4381         * Wait until the SCU boot is ready
4382         * This is done by iterating SCU_DELAY_ITERATION (10
4383         * microseconds each) up to SCU_DELAY_MAX (50) times.
4384         * This bit can be set in a random time that is less
4385         * than 500 microseconds after each power resume
4386         */
4387        retval2 = 0;
4388        delay_count = 0;
4389        while ((!retval2) && (delay_count < SCU_DELAY_MAX)) {
4390                retval2 = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR3_REG_ADDR);
4391                retval2 &= 0x00000008;
4392                if (!retval2) {
4393                        udelay(SCU_DELAY_ITERATION);
4394                        delay_count += 1;
4395                }
4396        }
4397
4398        if (!retval2) {
4399                dev_warn(&sep->pdev->dev, "scu boot bit not set at resume\n");
4400                return -EINVAL;
4401        }
4402
4403        /* Clear ICR register */
4404        sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF);
4405
4406        /* Set the IMR register - open only GPR 2 */
4407        sep_write_reg(sep, HW_HOST_IMR_REG_ADDR, (~(0x1 << 13)));
4408
4409        /* Read send/receive counters from SEP */
4410        sep->reply_ct = sep_read_reg(sep, HW_HOST_SEP_HOST_GPR2_REG_ADDR);
4411        sep->reply_ct &= 0x3FFFFFFF;
4412        sep->send_ct = sep->reply_ct;
4413
4414        return 0;
4415}
4416
4417/**
4418 * sep_pm_runtime_suspend - runtime suspend routine
4419 * @dev:        pointer to sep device
4420 *
4421 * Notes - revisit with more understanding of pm
4422 */
4423static int sep_pm_runtime_suspend(struct device *dev)
4424{
4425        struct sep_device *sep = sep_dev;
4426
4427        dev_dbg(&sep->pdev->dev, "pm runtime suspend called\n");
4428
4429        /* Clear ICR register */
4430        sep_write_reg(sep, HW_HOST_ICR_REG_ADDR, 0xFFFFFFFF);
4431        return 0;
4432}
4433
4434/**
4435 * sep_pm - power management for sep driver
4436 * @sep_pm_runtime_resume:      resume- no communication with cpu & main memory
4437 * @sep_pm_runtime_suspend:     suspend- no communication with cpu & main memory
4438 * @sep_pci_suspend:            suspend - main memory is still ON
4439 * @sep_pci_resume:             resume - main memory is still ON
4440 */
4441static const struct dev_pm_ops sep_pm = {
4442        .runtime_resume = sep_pm_runtime_resume,
4443        .runtime_suspend = sep_pm_runtime_suspend,
4444        .resume = sep_pci_resume,
4445        .suspend = sep_pci_suspend,
4446};
4447#endif /* SEP_ENABLE_RUNTIME_PM */
4448
4449/**
4450 * sep_pci_driver - registers this device with pci subsystem
4451 * @name:       name identifier for this driver
4452 * @sep_pci_id_tbl:     pointer to struct pci_device_id table
4453 * @sep_probe:  pointer to probe function in PCI driver
4454 * @sep_remove: pointer to remove function in PCI driver
4455 */
4456static struct pci_driver sep_pci_driver = {
4457#ifdef SEP_ENABLE_RUNTIME_PM
4458        .driver = {
4459                .pm = &sep_pm,
4460        },
4461#endif
4462        .name = "sep_sec_driver",
4463        .id_table = sep_pci_id_tbl,
4464        .probe = sep_probe,
4465        .remove = sep_remove
4466};
4467
4468module_pci_driver(sep_pci_driver);
4469MODULE_LICENSE("GPL");
4470
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.