linux/drivers/crypto/n2_core.c
<<
>>
Prefs
   1/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
   2 *
   3 * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
   4 */
   5
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8#include <linux/kernel.h>
   9#include <linux/module.h>
  10#include <linux/of.h>
  11#include <linux/of_device.h>
  12#include <linux/cpumask.h>
  13#include <linux/slab.h>
  14#include <linux/interrupt.h>
  15#include <linux/crypto.h>
  16#include <crypto/md5.h>
  17#include <crypto/sha.h>
  18#include <crypto/aes.h>
  19#include <crypto/des.h>
  20#include <linux/mutex.h>
  21#include <linux/delay.h>
  22#include <linux/sched.h>
  23
  24#include <crypto/internal/hash.h>
  25#include <crypto/scatterwalk.h>
  26#include <crypto/algapi.h>
  27
  28#include <asm/hypervisor.h>
  29#include <asm/mdesc.h>
  30
  31#include "n2_core.h"
  32
  33#define DRV_MODULE_NAME         "n2_crypto"
  34#define DRV_MODULE_VERSION      "0.2"
  35#define DRV_MODULE_RELDATE      "July 28, 2011"
  36
  37static char version[] =
  38        DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  39
  40MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
  41MODULE_DESCRIPTION("Niagara2 Crypto driver");
  42MODULE_LICENSE("GPL");
  43MODULE_VERSION(DRV_MODULE_VERSION);
  44
  45#define N2_CRA_PRIORITY         200
  46
  47static DEFINE_MUTEX(spu_lock);
  48
  49struct spu_queue {
  50        cpumask_t               sharing;
  51        unsigned long           qhandle;
  52
  53        spinlock_t              lock;
  54        u8                      q_type;
  55        void                    *q;
  56        unsigned long           head;
  57        unsigned long           tail;
  58        struct list_head        jobs;
  59
  60        unsigned long           devino;
  61
  62        char                    irq_name[32];
  63        unsigned int            irq;
  64
  65        struct list_head        list;
  66};
  67
  68static struct spu_queue **cpu_to_cwq;
  69static struct spu_queue **cpu_to_mau;
  70
  71static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
  72{
  73        if (q->q_type == HV_NCS_QTYPE_MAU) {
  74                off += MAU_ENTRY_SIZE;
  75                if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
  76                        off = 0;
  77        } else {
  78                off += CWQ_ENTRY_SIZE;
  79                if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
  80                        off = 0;
  81        }
  82        return off;
  83}
  84
  85struct n2_request_common {
  86        struct list_head        entry;
  87        unsigned int            offset;
  88};
  89#define OFFSET_NOT_RUNNING      (~(unsigned int)0)
  90
  91/* An async job request records the final tail value it used in
  92 * n2_request_common->offset, test to see if that offset is in
  93 * the range old_head, new_head, inclusive.
  94 */
  95static inline bool job_finished(struct spu_queue *q, unsigned int offset,
  96                                unsigned long old_head, unsigned long new_head)
  97{
  98        if (old_head <= new_head) {
  99                if (offset > old_head && offset <= new_head)
 100                        return true;
 101        } else {
 102                if (offset > old_head || offset <= new_head)
 103                        return true;
 104        }
 105        return false;
 106}
 107
 108/* When the HEAD marker is unequal to the actual HEAD, we get
 109 * a virtual device INO interrupt.  We should process the
 110 * completed CWQ entries and adjust the HEAD marker to clear
 111 * the IRQ.
 112 */
 113static irqreturn_t cwq_intr(int irq, void *dev_id)
 114{
 115        unsigned long off, new_head, hv_ret;
 116        struct spu_queue *q = dev_id;
 117
 118        pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
 119               smp_processor_id(), q->qhandle);
 120
 121        spin_lock(&q->lock);
 122
 123        hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
 124
 125        pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
 126               smp_processor_id(), new_head, hv_ret);
 127
 128        for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
 129                /* XXX ... XXX */
 130        }
 131
 132        hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
 133        if (hv_ret == HV_EOK)
 134                q->head = new_head;
 135
 136        spin_unlock(&q->lock);
 137
 138        return IRQ_HANDLED;
 139}
 140
 141static irqreturn_t mau_intr(int irq, void *dev_id)
 142{
 143        struct spu_queue *q = dev_id;
 144        unsigned long head, hv_ret;
 145
 146        spin_lock(&q->lock);
 147
 148        pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
 149               smp_processor_id(), q->qhandle);
 150
 151        hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
 152
 153        pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
 154               smp_processor_id(), head, hv_ret);
 155
 156        sun4v_ncs_sethead_marker(q->qhandle, head);
 157
 158        spin_unlock(&q->lock);
 159
 160        return IRQ_HANDLED;
 161}
 162
 163static void *spu_queue_next(struct spu_queue *q, void *cur)
 164{
 165        return q->q + spu_next_offset(q, cur - q->q);
 166}
 167
 168static int spu_queue_num_free(struct spu_queue *q)
 169{
 170        unsigned long head = q->head;
 171        unsigned long tail = q->tail;
 172        unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
 173        unsigned long diff;
 174
 175        if (head > tail)
 176                diff = head - tail;
 177        else
 178                diff = (end - tail) + head;
 179
 180        return (diff / CWQ_ENTRY_SIZE) - 1;
 181}
 182
 183static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
 184{
 185        int avail = spu_queue_num_free(q);
 186
 187        if (avail >= num_entries)
 188                return q->q + q->tail;
 189
 190        return NULL;
 191}
 192
 193static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
 194{
 195        unsigned long hv_ret, new_tail;
 196
 197        new_tail = spu_next_offset(q, last - q->q);
 198
 199        hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
 200        if (hv_ret == HV_EOK)
 201                q->tail = new_tail;
 202        return hv_ret;
 203}
 204
 205static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
 206                             int enc_type, int auth_type,
 207                             unsigned int hash_len,
 208                             bool sfas, bool sob, bool eob, bool encrypt,
 209                             int opcode)
 210{
 211        u64 word = (len - 1) & CONTROL_LEN;
 212
 213        word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
 214        word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
 215        word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
 216        if (sfas)
 217                word |= CONTROL_STORE_FINAL_AUTH_STATE;
 218        if (sob)
 219                word |= CONTROL_START_OF_BLOCK;
 220        if (eob)
 221                word |= CONTROL_END_OF_BLOCK;
 222        if (encrypt)
 223                word |= CONTROL_ENCRYPT;
 224        if (hmac_key_len)
 225                word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
 226        if (hash_len)
 227                word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
 228
 229        return word;
 230}
 231
 232#if 0
 233static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
 234{
 235        if (this_len >= 64 ||
 236            qp->head != qp->tail)
 237                return true;
 238        return false;
 239}
 240#endif
 241
 242struct n2_ahash_alg {
 243        struct list_head        entry;
 244        const char              *hash_zero;
 245        const u32               *hash_init;
 246        u8                      hw_op_hashsz;
 247        u8                      digest_size;
 248        u8                      auth_type;
 249        u8                      hmac_type;
 250        struct ahash_alg        alg;
 251};
 252
 253static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
 254{
 255        struct crypto_alg *alg = tfm->__crt_alg;
 256        struct ahash_alg *ahash_alg;
 257
 258        ahash_alg = container_of(alg, struct ahash_alg, halg.base);
 259
 260        return container_of(ahash_alg, struct n2_ahash_alg, alg);
 261}
 262
 263struct n2_hmac_alg {
 264        const char              *child_alg;
 265        struct n2_ahash_alg     derived;
 266};
 267
 268static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
 269{
 270        struct crypto_alg *alg = tfm->__crt_alg;
 271        struct ahash_alg *ahash_alg;
 272
 273        ahash_alg = container_of(alg, struct ahash_alg, halg.base);
 274
 275        return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
 276}
 277
 278struct n2_hash_ctx {
 279        struct crypto_ahash             *fallback_tfm;
 280};
 281
 282#define N2_HASH_KEY_MAX                 32 /* HW limit for all HMAC requests */
 283
 284struct n2_hmac_ctx {
 285        struct n2_hash_ctx              base;
 286
 287        struct crypto_shash             *child_shash;
 288
 289        int                             hash_key_len;
 290        unsigned char                   hash_key[N2_HASH_KEY_MAX];
 291};
 292
 293struct n2_hash_req_ctx {
 294        union {
 295                struct md5_state        md5;
 296                struct sha1_state       sha1;
 297                struct sha256_state     sha256;
 298        } u;
 299
 300        struct ahash_request            fallback_req;
 301};
 302
 303static int n2_hash_async_init(struct ahash_request *req)
 304{
 305        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 306        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 307        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 308
 309        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 310        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 311
 312        return crypto_ahash_init(&rctx->fallback_req);
 313}
 314
 315static int n2_hash_async_update(struct ahash_request *req)
 316{
 317        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 318        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 319        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 320
 321        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 322        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 323        rctx->fallback_req.nbytes = req->nbytes;
 324        rctx->fallback_req.src = req->src;
 325
 326        return crypto_ahash_update(&rctx->fallback_req);
 327}
 328
 329static int n2_hash_async_final(struct ahash_request *req)
 330{
 331        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 332        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 333        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 334
 335        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 336        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 337        rctx->fallback_req.result = req->result;
 338
 339        return crypto_ahash_final(&rctx->fallback_req);
 340}
 341
 342static int n2_hash_async_finup(struct ahash_request *req)
 343{
 344        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 345        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 346        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 347
 348        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 349        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 350        rctx->fallback_req.nbytes = req->nbytes;
 351        rctx->fallback_req.src = req->src;
 352        rctx->fallback_req.result = req->result;
 353
 354        return crypto_ahash_finup(&rctx->fallback_req);
 355}
 356
 357static int n2_hash_cra_init(struct crypto_tfm *tfm)
 358{
 359        const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
 360        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 361        struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 362        struct crypto_ahash *fallback_tfm;
 363        int err;
 364
 365        fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 366                                          CRYPTO_ALG_NEED_FALLBACK);
 367        if (IS_ERR(fallback_tfm)) {
 368                pr_warning("Fallback driver '%s' could not be loaded!\n",
 369                           fallback_driver_name);
 370                err = PTR_ERR(fallback_tfm);
 371                goto out;
 372        }
 373
 374        crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 375                                         crypto_ahash_reqsize(fallback_tfm)));
 376
 377        ctx->fallback_tfm = fallback_tfm;
 378        return 0;
 379
 380out:
 381        return err;
 382}
 383
 384static void n2_hash_cra_exit(struct crypto_tfm *tfm)
 385{
 386        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 387        struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 388
 389        crypto_free_ahash(ctx->fallback_tfm);
 390}
 391
 392static int n2_hmac_cra_init(struct crypto_tfm *tfm)
 393{
 394        const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
 395        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 396        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
 397        struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
 398        struct crypto_ahash *fallback_tfm;
 399        struct crypto_shash *child_shash;
 400        int err;
 401
 402        fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 403                                          CRYPTO_ALG_NEED_FALLBACK);
 404        if (IS_ERR(fallback_tfm)) {
 405                pr_warning("Fallback driver '%s' could not be loaded!\n",
 406                           fallback_driver_name);
 407                err = PTR_ERR(fallback_tfm);
 408                goto out;
 409        }
 410
 411        child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
 412        if (IS_ERR(child_shash)) {
 413                pr_warning("Child shash '%s' could not be loaded!\n",
 414                           n2alg->child_alg);
 415                err = PTR_ERR(child_shash);
 416                goto out_free_fallback;
 417        }
 418
 419        crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 420                                         crypto_ahash_reqsize(fallback_tfm)));
 421
 422        ctx->child_shash = child_shash;
 423        ctx->base.fallback_tfm = fallback_tfm;
 424        return 0;
 425
 426out_free_fallback:
 427        crypto_free_ahash(fallback_tfm);
 428
 429out:
 430        return err;
 431}
 432
 433static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
 434{
 435        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 436        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
 437
 438        crypto_free_ahash(ctx->base.fallback_tfm);
 439        crypto_free_shash(ctx->child_shash);
 440}
 441
 442static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
 443                                unsigned int keylen)
 444{
 445        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
 446        struct crypto_shash *child_shash = ctx->child_shash;
 447        struct crypto_ahash *fallback_tfm;
 448        SHASH_DESC_ON_STACK(shash, child_shash);
 449        int err, bs, ds;
 450
 451        fallback_tfm = ctx->base.fallback_tfm;
 452        err = crypto_ahash_setkey(fallback_tfm, key, keylen);
 453        if (err)
 454                return err;
 455
 456        shash->tfm = child_shash;
 457        shash->flags = crypto_ahash_get_flags(tfm) &
 458                CRYPTO_TFM_REQ_MAY_SLEEP;
 459
 460        bs = crypto_shash_blocksize(child_shash);
 461        ds = crypto_shash_digestsize(child_shash);
 462        BUG_ON(ds > N2_HASH_KEY_MAX);
 463        if (keylen > bs) {
 464                err = crypto_shash_digest(shash, key, keylen,
 465                                          ctx->hash_key);
 466                if (err)
 467                        return err;
 468                keylen = ds;
 469        } else if (keylen <= N2_HASH_KEY_MAX)
 470                memcpy(ctx->hash_key, key, keylen);
 471
 472        ctx->hash_key_len = keylen;
 473
 474        return err;
 475}
 476
 477static unsigned long wait_for_tail(struct spu_queue *qp)
 478{
 479        unsigned long head, hv_ret;
 480
 481        do {
 482                hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
 483                if (hv_ret != HV_EOK) {
 484                        pr_err("Hypervisor error on gethead\n");
 485                        break;
 486                }
 487                if (head == qp->tail) {
 488                        qp->head = head;
 489                        break;
 490                }
 491        } while (1);
 492        return hv_ret;
 493}
 494
 495static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
 496                                              struct cwq_initial_entry *ent)
 497{
 498        unsigned long hv_ret = spu_queue_submit(qp, ent);
 499
 500        if (hv_ret == HV_EOK)
 501                hv_ret = wait_for_tail(qp);
 502
 503        return hv_ret;
 504}
 505
 506static int n2_do_async_digest(struct ahash_request *req,
 507                              unsigned int auth_type, unsigned int digest_size,
 508                              unsigned int result_size, void *hash_loc,
 509                              unsigned long auth_key, unsigned int auth_key_len)
 510{
 511        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 512        struct cwq_initial_entry *ent;
 513        struct crypto_hash_walk walk;
 514        struct spu_queue *qp;
 515        unsigned long flags;
 516        int err = -ENODEV;
 517        int nbytes, cpu;
 518
 519        /* The total effective length of the operation may not
 520         * exceed 2^16.
 521         */
 522        if (unlikely(req->nbytes > (1 << 16))) {
 523                struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 524                struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 525
 526                ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 527                rctx->fallback_req.base.flags =
 528                        req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 529                rctx->fallback_req.nbytes = req->nbytes;
 530                rctx->fallback_req.src = req->src;
 531                rctx->fallback_req.result = req->result;
 532
 533                return crypto_ahash_digest(&rctx->fallback_req);
 534        }
 535
 536        nbytes = crypto_hash_walk_first(req, &walk);
 537
 538        cpu = get_cpu();
 539        qp = cpu_to_cwq[cpu];
 540        if (!qp)
 541                goto out;
 542
 543        spin_lock_irqsave(&qp->lock, flags);
 544
 545        /* XXX can do better, improve this later by doing a by-hand scatterlist
 546         * XXX walk, etc.
 547         */
 548        ent = qp->q + qp->tail;
 549
 550        ent->control = control_word_base(nbytes, auth_key_len, 0,
 551                                         auth_type, digest_size,
 552                                         false, true, false, false,
 553                                         OPCODE_INPLACE_BIT |
 554                                         OPCODE_AUTH_MAC);
 555        ent->src_addr = __pa(walk.data);
 556        ent->auth_key_addr = auth_key;
 557        ent->auth_iv_addr = __pa(hash_loc);
 558        ent->final_auth_state_addr = 0UL;
 559        ent->enc_key_addr = 0UL;
 560        ent->enc_iv_addr = 0UL;
 561        ent->dest_addr = __pa(hash_loc);
 562
 563        nbytes = crypto_hash_walk_done(&walk, 0);
 564        while (nbytes > 0) {
 565                ent = spu_queue_next(qp, ent);
 566
 567                ent->control = (nbytes - 1);
 568                ent->src_addr = __pa(walk.data);
 569                ent->auth_key_addr = 0UL;
 570                ent->auth_iv_addr = 0UL;
 571                ent->final_auth_state_addr = 0UL;
 572                ent->enc_key_addr = 0UL;
 573                ent->enc_iv_addr = 0UL;
 574                ent->dest_addr = 0UL;
 575
 576                nbytes = crypto_hash_walk_done(&walk, 0);
 577        }
 578        ent->control |= CONTROL_END_OF_BLOCK;
 579
 580        if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
 581                err = -EINVAL;
 582        else
 583                err = 0;
 584
 585        spin_unlock_irqrestore(&qp->lock, flags);
 586
 587        if (!err)
 588                memcpy(req->result, hash_loc, result_size);
 589out:
 590        put_cpu();
 591
 592        return err;
 593}
 594
 595static int n2_hash_async_digest(struct ahash_request *req)
 596{
 597        struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
 598        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 599        int ds;
 600
 601        ds = n2alg->digest_size;
 602        if (unlikely(req->nbytes == 0)) {
 603                memcpy(req->result, n2alg->hash_zero, ds);
 604                return 0;
 605        }
 606        memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
 607
 608        return n2_do_async_digest(req, n2alg->auth_type,
 609                                  n2alg->hw_op_hashsz, ds,
 610                                  &rctx->u, 0UL, 0);
 611}
 612
 613static int n2_hmac_async_digest(struct ahash_request *req)
 614{
 615        struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
 616        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 617        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 618        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
 619        int ds;
 620
 621        ds = n2alg->derived.digest_size;
 622        if (unlikely(req->nbytes == 0) ||
 623            unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
 624                struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 625                struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 626
 627                ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 628                rctx->fallback_req.base.flags =
 629                        req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 630                rctx->fallback_req.nbytes = req->nbytes;
 631                rctx->fallback_req.src = req->src;
 632                rctx->fallback_req.result = req->result;
 633
 634                return crypto_ahash_digest(&rctx->fallback_req);
 635        }
 636        memcpy(&rctx->u, n2alg->derived.hash_init,
 637               n2alg->derived.hw_op_hashsz);
 638
 639        return n2_do_async_digest(req, n2alg->derived.hmac_type,
 640                                  n2alg->derived.hw_op_hashsz, ds,
 641                                  &rctx->u,
 642                                  __pa(&ctx->hash_key),
 643                                  ctx->hash_key_len);
 644}
 645
 646struct n2_cipher_context {
 647        int                     key_len;
 648        int                     enc_type;
 649        union {
 650                u8              aes[AES_MAX_KEY_SIZE];
 651                u8              des[DES_KEY_SIZE];
 652                u8              des3[3 * DES_KEY_SIZE];
 653                u8              arc4[258]; /* S-box, X, Y */
 654        } key;
 655};
 656
 657#define N2_CHUNK_ARR_LEN        16
 658
 659struct n2_crypto_chunk {
 660        struct list_head        entry;
 661        unsigned long           iv_paddr : 44;
 662        unsigned long           arr_len : 20;
 663        unsigned long           dest_paddr;
 664        unsigned long           dest_final;
 665        struct {
 666                unsigned long   src_paddr : 44;
 667                unsigned long   src_len : 20;
 668        } arr[N2_CHUNK_ARR_LEN];
 669};
 670
 671struct n2_request_context {
 672        struct ablkcipher_walk  walk;
 673        struct list_head        chunk_list;
 674        struct n2_crypto_chunk  chunk;
 675        u8                      temp_iv[16];
 676};
 677
 678/* The SPU allows some level of flexibility for partial cipher blocks
 679 * being specified in a descriptor.
 680 *
 681 * It merely requires that every descriptor's length field is at least
 682 * as large as the cipher block size.  This means that a cipher block
 683 * can span at most 2 descriptors.  However, this does not allow a
 684 * partial block to span into the final descriptor as that would
 685 * violate the rule (since every descriptor's length must be at lest
 686 * the block size).  So, for example, assuming an 8 byte block size:
 687 *
 688 *      0xe --> 0xa --> 0x8
 689 *
 690 * is a valid length sequence, whereas:
 691 *
 692 *      0xe --> 0xb --> 0x7
 693 *
 694 * is not a valid sequence.
 695 */
 696
 697struct n2_cipher_alg {
 698        struct list_head        entry;
 699        u8                      enc_type;
 700        struct crypto_alg       alg;
 701};
 702
 703static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
 704{
 705        struct crypto_alg *alg = tfm->__crt_alg;
 706
 707        return container_of(alg, struct n2_cipher_alg, alg);
 708}
 709
 710struct n2_cipher_request_context {
 711        struct ablkcipher_walk  walk;
 712};
 713
 714static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 715                         unsigned int keylen)
 716{
 717        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 718        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 719        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 720
 721        ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
 722
 723        switch (keylen) {
 724        case AES_KEYSIZE_128:
 725                ctx->enc_type |= ENC_TYPE_ALG_AES128;
 726                break;
 727        case AES_KEYSIZE_192:
 728                ctx->enc_type |= ENC_TYPE_ALG_AES192;
 729                break;
 730        case AES_KEYSIZE_256:
 731                ctx->enc_type |= ENC_TYPE_ALG_AES256;
 732                break;
 733        default:
 734                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 735                return -EINVAL;
 736        }
 737
 738        ctx->key_len = keylen;
 739        memcpy(ctx->key.aes, key, keylen);
 740        return 0;
 741}
 742
 743static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 744                         unsigned int keylen)
 745{
 746        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 747        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 748        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 749        u32 tmp[DES_EXPKEY_WORDS];
 750        int err;
 751
 752        ctx->enc_type = n2alg->enc_type;
 753
 754        if (keylen != DES_KEY_SIZE) {
 755                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 756                return -EINVAL;
 757        }
 758
 759        err = des_ekey(tmp, key);
 760        if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 761                tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 762                return -EINVAL;
 763        }
 764
 765        ctx->key_len = keylen;
 766        memcpy(ctx->key.des, key, keylen);
 767        return 0;
 768}
 769
 770static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 771                          unsigned int keylen)
 772{
 773        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 774        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 775        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 776
 777        ctx->enc_type = n2alg->enc_type;
 778
 779        if (keylen != (3 * DES_KEY_SIZE)) {
 780                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 781                return -EINVAL;
 782        }
 783        ctx->key_len = keylen;
 784        memcpy(ctx->key.des3, key, keylen);
 785        return 0;
 786}
 787
 788static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 789                          unsigned int keylen)
 790{
 791        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 792        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 793        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 794        u8 *s = ctx->key.arc4;
 795        u8 *x = s + 256;
 796        u8 *y = x + 1;
 797        int i, j, k;
 798
 799        ctx->enc_type = n2alg->enc_type;
 800
 801        j = k = 0;
 802        *x = 0;
 803        *y = 0;
 804        for (i = 0; i < 256; i++)
 805                s[i] = i;
 806        for (i = 0; i < 256; i++) {
 807                u8 a = s[i];
 808                j = (j + key[k] + a) & 0xff;
 809                s[i] = s[j];
 810                s[j] = a;
 811                if (++k >= keylen)
 812                        k = 0;
 813        }
 814
 815        return 0;
 816}
 817
 818static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
 819{
 820        int this_len = nbytes;
 821
 822        this_len -= (nbytes & (block_size - 1));
 823        return this_len > (1 << 16) ? (1 << 16) : this_len;
 824}
 825
 826static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
 827                            struct spu_queue *qp, bool encrypt)
 828{
 829        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 830        struct cwq_initial_entry *ent;
 831        bool in_place;
 832        int i;
 833
 834        ent = spu_queue_alloc(qp, cp->arr_len);
 835        if (!ent) {
 836                pr_info("queue_alloc() of %d fails\n",
 837                        cp->arr_len);
 838                return -EBUSY;
 839        }
 840
 841        in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
 842
 843        ent->control = control_word_base(cp->arr[0].src_len,
 844                                         0, ctx->enc_type, 0, 0,
 845                                         false, true, false, encrypt,
 846                                         OPCODE_ENCRYPT |
 847                                         (in_place ? OPCODE_INPLACE_BIT : 0));
 848        ent->src_addr = cp->arr[0].src_paddr;
 849        ent->auth_key_addr = 0UL;
 850        ent->auth_iv_addr = 0UL;
 851        ent->final_auth_state_addr = 0UL;
 852        ent->enc_key_addr = __pa(&ctx->key);
 853        ent->enc_iv_addr = cp->iv_paddr;
 854        ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
 855
 856        for (i = 1; i < cp->arr_len; i++) {
 857                ent = spu_queue_next(qp, ent);
 858
 859                ent->control = cp->arr[i].src_len - 1;
 860                ent->src_addr = cp->arr[i].src_paddr;
 861                ent->auth_key_addr = 0UL;
 862                ent->auth_iv_addr = 0UL;
 863                ent->final_auth_state_addr = 0UL;
 864                ent->enc_key_addr = 0UL;
 865                ent->enc_iv_addr = 0UL;
 866                ent->dest_addr = 0UL;
 867        }
 868        ent->control |= CONTROL_END_OF_BLOCK;
 869
 870        return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
 871}
 872
 873static int n2_compute_chunks(struct ablkcipher_request *req)
 874{
 875        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 876        struct ablkcipher_walk *walk = &rctx->walk;
 877        struct n2_crypto_chunk *chunk;
 878        unsigned long dest_prev;
 879        unsigned int tot_len;
 880        bool prev_in_place;
 881        int err, nbytes;
 882
 883        ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
 884        err = ablkcipher_walk_phys(req, walk);
 885        if (err)
 886                return err;
 887
 888        INIT_LIST_HEAD(&rctx->chunk_list);
 889
 890        chunk = &rctx->chunk;
 891        INIT_LIST_HEAD(&chunk->entry);
 892
 893        chunk->iv_paddr = 0UL;
 894        chunk->arr_len = 0;
 895        chunk->dest_paddr = 0UL;
 896
 897        prev_in_place = false;
 898        dest_prev = ~0UL;
 899        tot_len = 0;
 900
 901        while ((nbytes = walk->nbytes) != 0) {
 902                unsigned long dest_paddr, src_paddr;
 903                bool in_place;
 904                int this_len;
 905
 906                src_paddr = (page_to_phys(walk->src.page) +
 907                             walk->src.offset);
 908                dest_paddr = (page_to_phys(walk->dst.page) +
 909                              walk->dst.offset);
 910                in_place = (src_paddr == dest_paddr);
 911                this_len = cipher_descriptor_len(nbytes, walk->blocksize);
 912
 913                if (chunk->arr_len != 0) {
 914                        if (in_place != prev_in_place ||
 915                            (!prev_in_place &&
 916                             dest_paddr != dest_prev) ||
 917                            chunk->arr_len == N2_CHUNK_ARR_LEN ||
 918                            tot_len + this_len > (1 << 16)) {
 919                                chunk->dest_final = dest_prev;
 920                                list_add_tail(&chunk->entry,
 921                                              &rctx->chunk_list);
 922                                chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
 923                                if (!chunk) {
 924                                        err = -ENOMEM;
 925                                        break;
 926                                }
 927                                INIT_LIST_HEAD(&chunk->entry);
 928                        }
 929                }
 930                if (chunk->arr_len == 0) {
 931                        chunk->dest_paddr = dest_paddr;
 932                        tot_len = 0;
 933                }
 934                chunk->arr[chunk->arr_len].src_paddr = src_paddr;
 935                chunk->arr[chunk->arr_len].src_len = this_len;
 936                chunk->arr_len++;
 937
 938                dest_prev = dest_paddr + this_len;
 939                prev_in_place = in_place;
 940                tot_len += this_len;
 941
 942                err = ablkcipher_walk_done(req, walk, nbytes - this_len);
 943                if (err)
 944                        break;
 945        }
 946        if (!err && chunk->arr_len != 0) {
 947                chunk->dest_final = dest_prev;
 948                list_add_tail(&chunk->entry, &rctx->chunk_list);
 949        }
 950
 951        return err;
 952}
 953
 954static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
 955{
 956        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 957        struct n2_crypto_chunk *c, *tmp;
 958
 959        if (final_iv)
 960                memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
 961
 962        ablkcipher_walk_complete(&rctx->walk);
 963        list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 964                list_del(&c->entry);
 965                if (unlikely(c != &rctx->chunk))
 966                        kfree(c);
 967        }
 968
 969}
 970
 971static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
 972{
 973        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 974        struct crypto_tfm *tfm = req->base.tfm;
 975        int err = n2_compute_chunks(req);
 976        struct n2_crypto_chunk *c, *tmp;
 977        unsigned long flags, hv_ret;
 978        struct spu_queue *qp;
 979
 980        if (err)
 981                return err;
 982
 983        qp = cpu_to_cwq[get_cpu()];
 984        err = -ENODEV;
 985        if (!qp)
 986                goto out;
 987
 988        spin_lock_irqsave(&qp->lock, flags);
 989
 990        list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 991                err = __n2_crypt_chunk(tfm, c, qp, encrypt);
 992                if (err)
 993                        break;
 994                list_del(&c->entry);
 995                if (unlikely(c != &rctx->chunk))
 996                        kfree(c);
 997        }
 998        if (!err) {
 999                hv_ret = wait_for_tail(qp);
1000                if (hv_ret != HV_EOK)
1001                        err = -EINVAL;
1002        }
1003
1004        spin_unlock_irqrestore(&qp->lock, flags);
1005
1006out:
1007        put_cpu();
1008
1009        n2_chunk_complete(req, NULL);
1010        return err;
1011}
1012
1013static int n2_encrypt_ecb(struct ablkcipher_request *req)
1014{
1015        return n2_do_ecb(req, true);
1016}
1017
1018static int n2_decrypt_ecb(struct ablkcipher_request *req)
1019{
1020        return n2_do_ecb(req, false);
1021}
1022
1023static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
1024{
1025        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
1026        struct crypto_tfm *tfm = req->base.tfm;
1027        unsigned long flags, hv_ret, iv_paddr;
1028        int err = n2_compute_chunks(req);
1029        struct n2_crypto_chunk *c, *tmp;
1030        struct spu_queue *qp;
1031        void *final_iv_addr;
1032
1033        final_iv_addr = NULL;
1034
1035        if (err)
1036                return err;
1037
1038        qp = cpu_to_cwq[get_cpu()];
1039        err = -ENODEV;
1040        if (!qp)
1041                goto out;
1042
1043        spin_lock_irqsave(&qp->lock, flags);
1044
1045        if (encrypt) {
1046                iv_paddr = __pa(rctx->walk.iv);
1047                list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
1048                                         entry) {
1049                        c->iv_paddr = iv_paddr;
1050                        err = __n2_crypt_chunk(tfm, c, qp, true);
1051                        if (err)
1052                                break;
1053                        iv_paddr = c->dest_final - rctx->walk.blocksize;
1054                        list_del(&c->entry);
1055                        if (unlikely(c != &rctx->chunk))
1056                                kfree(c);
1057                }
1058                final_iv_addr = __va(iv_paddr);
1059        } else {
1060                list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1061                                                 entry) {
1062                        if (c == &rctx->chunk) {
1063                                iv_paddr = __pa(rctx->walk.iv);
1064                        } else {
1065                                iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1066                                            tmp->arr[tmp->arr_len-1].src_len -
1067                                            rctx->walk.blocksize);
1068                        }
1069                        if (!final_iv_addr) {
1070                                unsigned long pa;
1071
1072                                pa = (c->arr[c->arr_len-1].src_paddr +
1073                                      c->arr[c->arr_len-1].src_len -
1074                                      rctx->walk.blocksize);
1075                                final_iv_addr = rctx->temp_iv;
1076                                memcpy(rctx->temp_iv, __va(pa),
1077                                       rctx->walk.blocksize);
1078                        }
1079                        c->iv_paddr = iv_paddr;
1080                        err = __n2_crypt_chunk(tfm, c, qp, false);
1081                        if (err)
1082                                break;
1083                        list_del(&c->entry);
1084                        if (unlikely(c != &rctx->chunk))
1085                                kfree(c);
1086                }
1087        }
1088        if (!err) {
1089                hv_ret = wait_for_tail(qp);
1090                if (hv_ret != HV_EOK)
1091                        err = -EINVAL;
1092        }
1093
1094        spin_unlock_irqrestore(&qp->lock, flags);
1095
1096out:
1097        put_cpu();
1098
1099        n2_chunk_complete(req, err ? NULL : final_iv_addr);
1100        return err;
1101}
1102
1103static int n2_encrypt_chaining(struct ablkcipher_request *req)
1104{
1105        return n2_do_chaining(req, true);
1106}
1107
1108static int n2_decrypt_chaining(struct ablkcipher_request *req)
1109{
1110        return n2_do_chaining(req, false);
1111}
1112
1113struct n2_cipher_tmpl {
1114        const char              *name;
1115        const char              *drv_name;
1116        u8                      block_size;
1117        u8                      enc_type;
1118        struct ablkcipher_alg   ablkcipher;
1119};
1120
1121static const struct n2_cipher_tmpl cipher_tmpls[] = {
1122        /* ARC4: only ECB is supported (chaining bits ignored) */
1123        {       .name           = "ecb(arc4)",
1124                .drv_name       = "ecb-arc4",
1125                .block_size     = 1,
1126                .enc_type       = (ENC_TYPE_ALG_RC4_STREAM |
1127                                   ENC_TYPE_CHAINING_ECB),
1128                .ablkcipher     = {
1129                        .min_keysize    = 1,
1130                        .max_keysize    = 256,
1131                        .setkey         = n2_arc4_setkey,
1132                        .encrypt        = n2_encrypt_ecb,
1133                        .decrypt        = n2_decrypt_ecb,
1134                },
1135        },
1136
1137        /* DES: ECB CBC and CFB are supported */
1138        {       .name           = "ecb(des)",
1139                .drv_name       = "ecb-des",
1140                .block_size     = DES_BLOCK_SIZE,
1141                .enc_type       = (ENC_TYPE_ALG_DES |
1142                                   ENC_TYPE_CHAINING_ECB),
1143                .ablkcipher     = {
1144                        .min_keysize    = DES_KEY_SIZE,
1145                        .max_keysize    = DES_KEY_SIZE,
1146                        .setkey         = n2_des_setkey,
1147                        .encrypt        = n2_encrypt_ecb,
1148                        .decrypt        = n2_decrypt_ecb,
1149                },
1150        },
1151        {       .name           = "cbc(des)",
1152                .drv_name       = "cbc-des",
1153                .block_size     = DES_BLOCK_SIZE,
1154                .enc_type       = (ENC_TYPE_ALG_DES |
1155                                   ENC_TYPE_CHAINING_CBC),
1156                .ablkcipher     = {
1157                        .ivsize         = DES_BLOCK_SIZE,
1158                        .min_keysize    = DES_KEY_SIZE,
1159                        .max_keysize    = DES_KEY_SIZE,
1160                        .setkey         = n2_des_setkey,
1161                        .encrypt        = n2_encrypt_chaining,
1162                        .decrypt        = n2_decrypt_chaining,
1163                },
1164        },
1165        {       .name           = "cfb(des)",
1166                .drv_name       = "cfb-des",
1167                .block_size     = DES_BLOCK_SIZE,
1168                .enc_type       = (ENC_TYPE_ALG_DES |
1169                                   ENC_TYPE_CHAINING_CFB),
1170                .ablkcipher     = {
1171                        .min_keysize    = DES_KEY_SIZE,
1172                        .max_keysize    = DES_KEY_SIZE,
1173                        .setkey         = n2_des_setkey,
1174                        .encrypt        = n2_encrypt_chaining,
1175                        .decrypt        = n2_decrypt_chaining,
1176                },
1177        },
1178
1179        /* 3DES: ECB CBC and CFB are supported */
1180        {       .name           = "ecb(des3_ede)",
1181                .drv_name       = "ecb-3des",
1182                .block_size     = DES_BLOCK_SIZE,
1183                .enc_type       = (ENC_TYPE_ALG_3DES |
1184                                   ENC_TYPE_CHAINING_ECB),
1185                .ablkcipher     = {
1186                        .min_keysize    = 3 * DES_KEY_SIZE,
1187                        .max_keysize    = 3 * DES_KEY_SIZE,
1188                        .setkey         = n2_3des_setkey,
1189                        .encrypt        = n2_encrypt_ecb,
1190                        .decrypt        = n2_decrypt_ecb,
1191                },
1192        },
1193        {       .name           = "cbc(des3_ede)",
1194                .drv_name       = "cbc-3des",
1195                .block_size     = DES_BLOCK_SIZE,
1196                .enc_type       = (ENC_TYPE_ALG_3DES |
1197                                   ENC_TYPE_CHAINING_CBC),
1198                .ablkcipher     = {
1199                        .ivsize         = DES_BLOCK_SIZE,
1200                        .min_keysize    = 3 * DES_KEY_SIZE,
1201                        .max_keysize    = 3 * DES_KEY_SIZE,
1202                        .setkey         = n2_3des_setkey,
1203                        .encrypt        = n2_encrypt_chaining,
1204                        .decrypt        = n2_decrypt_chaining,
1205                },
1206        },
1207        {       .name           = "cfb(des3_ede)",
1208                .drv_name       = "cfb-3des",
1209                .block_size     = DES_BLOCK_SIZE,
1210                .enc_type       = (ENC_TYPE_ALG_3DES |
1211                                   ENC_TYPE_CHAINING_CFB),
1212                .ablkcipher     = {
1213                        .min_keysize    = 3 * DES_KEY_SIZE,
1214                        .max_keysize    = 3 * DES_KEY_SIZE,
1215                        .setkey         = n2_3des_setkey,
1216                        .encrypt        = n2_encrypt_chaining,
1217                        .decrypt        = n2_decrypt_chaining,
1218                },
1219        },
1220        /* AES: ECB CBC and CTR are supported */
1221        {       .name           = "ecb(aes)",
1222                .drv_name       = "ecb-aes",
1223                .block_size     = AES_BLOCK_SIZE,
1224                .enc_type       = (ENC_TYPE_ALG_AES128 |
1225                                   ENC_TYPE_CHAINING_ECB),
1226                .ablkcipher     = {
1227                        .min_keysize    = AES_MIN_KEY_SIZE,
1228                        .max_keysize    = AES_MAX_KEY_SIZE,
1229                        .setkey         = n2_aes_setkey,
1230                        .encrypt        = n2_encrypt_ecb,
1231                        .decrypt        = n2_decrypt_ecb,
1232                },
1233        },
1234        {       .name           = "cbc(aes)",
1235                .drv_name       = "cbc-aes",
1236                .block_size     = AES_BLOCK_SIZE,
1237                .enc_type       = (ENC_TYPE_ALG_AES128 |
1238                                   ENC_TYPE_CHAINING_CBC),
1239                .ablkcipher     = {
1240                        .ivsize         = AES_BLOCK_SIZE,
1241                        .min_keysize    = AES_MIN_KEY_SIZE,
1242                        .max_keysize    = AES_MAX_KEY_SIZE,
1243                        .setkey         = n2_aes_setkey,
1244                        .encrypt        = n2_encrypt_chaining,
1245                        .decrypt        = n2_decrypt_chaining,
1246                },
1247        },
1248        {       .name           = "ctr(aes)",
1249                .drv_name       = "ctr-aes",
1250                .block_size     = AES_BLOCK_SIZE,
1251                .enc_type       = (ENC_TYPE_ALG_AES128 |
1252                                   ENC_TYPE_CHAINING_COUNTER),
1253                .ablkcipher     = {
1254                        .ivsize         = AES_BLOCK_SIZE,
1255                        .min_keysize    = AES_MIN_KEY_SIZE,
1256                        .max_keysize    = AES_MAX_KEY_SIZE,
1257                        .setkey         = n2_aes_setkey,
1258                        .encrypt        = n2_encrypt_chaining,
1259                        .decrypt        = n2_encrypt_chaining,
1260                },
1261        },
1262
1263};
1264#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1265
1266static LIST_HEAD(cipher_algs);
1267
1268struct n2_hash_tmpl {
1269        const char      *name;
1270        const char      *hash_zero;
1271        const u32       *hash_init;
1272        u8              hw_op_hashsz;
1273        u8              digest_size;
1274        u8              block_size;
1275        u8              auth_type;
1276        u8              hmac_type;
1277};
1278
1279static const char md5_zero[MD5_DIGEST_SIZE] = {
1280        0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
1281        0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
1282};
1283static const u32 md5_init[MD5_HASH_WORDS] = {
1284        cpu_to_le32(0x67452301),
1285        cpu_to_le32(0xefcdab89),
1286        cpu_to_le32(0x98badcfe),
1287        cpu_to_le32(0x10325476),
1288};
1289static const char sha1_zero[SHA1_DIGEST_SIZE] = {
1290        0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
1291        0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
1292        0x07, 0x09
1293};
1294static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
1295        SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
1296};
1297static const char sha256_zero[SHA256_DIGEST_SIZE] = {
1298        0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
1299        0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
1300        0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
1301        0x1b, 0x78, 0x52, 0xb8, 0x55
1302};
1303static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
1304        SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
1305        SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
1306};
1307static const char sha224_zero[SHA224_DIGEST_SIZE] = {
1308        0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
1309        0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
1310        0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
1311        0x2f
1312};
1313static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
1314        SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
1315        SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
1316};
1317
1318static const struct n2_hash_tmpl hash_tmpls[] = {
1319        { .name         = "md5",
1320          .hash_zero    = md5_zero,
1321          .hash_init    = md5_init,
1322          .auth_type    = AUTH_TYPE_MD5,
1323          .hmac_type    = AUTH_TYPE_HMAC_MD5,
1324          .hw_op_hashsz = MD5_DIGEST_SIZE,
1325          .digest_size  = MD5_DIGEST_SIZE,
1326          .block_size   = MD5_HMAC_BLOCK_SIZE },
1327        { .name         = "sha1",
1328          .hash_zero    = sha1_zero,
1329          .hash_init    = sha1_init,
1330          .auth_type    = AUTH_TYPE_SHA1,
1331          .hmac_type    = AUTH_TYPE_HMAC_SHA1,
1332          .hw_op_hashsz = SHA1_DIGEST_SIZE,
1333          .digest_size  = SHA1_DIGEST_SIZE,
1334          .block_size   = SHA1_BLOCK_SIZE },
1335        { .name         = "sha256",
1336          .hash_zero    = sha256_zero,
1337          .hash_init    = sha256_init,
1338          .auth_type    = AUTH_TYPE_SHA256,
1339          .hmac_type    = AUTH_TYPE_HMAC_SHA256,
1340          .hw_op_hashsz = SHA256_DIGEST_SIZE,
1341          .digest_size  = SHA256_DIGEST_SIZE,
1342          .block_size   = SHA256_BLOCK_SIZE },
1343        { .name         = "sha224",
1344          .hash_zero    = sha224_zero,
1345          .hash_init    = sha224_init,
1346          .auth_type    = AUTH_TYPE_SHA256,
1347          .hmac_type    = AUTH_TYPE_RESERVED,
1348          .hw_op_hashsz = SHA256_DIGEST_SIZE,
1349          .digest_size  = SHA224_DIGEST_SIZE,
1350          .block_size   = SHA224_BLOCK_SIZE },
1351};
1352#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1353
1354static LIST_HEAD(ahash_algs);
1355static LIST_HEAD(hmac_algs);
1356
1357static int algs_registered;
1358
1359static void __n2_unregister_algs(void)
1360{
1361        struct n2_cipher_alg *cipher, *cipher_tmp;
1362        struct n2_ahash_alg *alg, *alg_tmp;
1363        struct n2_hmac_alg *hmac, *hmac_tmp;
1364
1365        list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1366                crypto_unregister_alg(&cipher->alg);
1367                list_del(&cipher->entry);
1368                kfree(cipher);
1369        }
1370        list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
1371                crypto_unregister_ahash(&hmac->derived.alg);
1372                list_del(&hmac->derived.entry);
1373                kfree(hmac);
1374        }
1375        list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1376                crypto_unregister_ahash(&alg->alg);
1377                list_del(&alg->entry);
1378                kfree(alg);
1379        }
1380}
1381
1382static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1383{
1384        tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1385        return 0;
1386}
1387
1388static int __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
1389{
1390        struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1391        struct crypto_alg *alg;
1392        int err;
1393
1394        if (!p)
1395                return -ENOMEM;
1396
1397        alg = &p->alg;
1398
1399        snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1400        snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1401        alg->cra_priority = N2_CRA_PRIORITY;
1402        alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1403                         CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
1404        alg->cra_blocksize = tmpl->block_size;
1405        p->enc_type = tmpl->enc_type;
1406        alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1407        alg->cra_type = &crypto_ablkcipher_type;
1408        alg->cra_u.ablkcipher = tmpl->ablkcipher;
1409        alg->cra_init = n2_cipher_cra_init;
1410        alg->cra_module = THIS_MODULE;
1411
1412        list_add(&p->entry, &cipher_algs);
1413        err = crypto_register_alg(alg);
1414        if (err) {
1415                pr_err("%s alg registration failed\n", alg->cra_name);
1416                list_del(&p->entry);
1417                kfree(p);
1418        } else {
1419                pr_info("%s alg registered\n", alg->cra_name);
1420        }
1421        return err;
1422}
1423
1424static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
1425{
1426        struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1427        struct ahash_alg *ahash;
1428        struct crypto_alg *base;
1429        int err;
1430
1431        if (!p)
1432                return -ENOMEM;
1433
1434        p->child_alg = n2ahash->alg.halg.base.cra_name;
1435        memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
1436        INIT_LIST_HEAD(&p->derived.entry);
1437
1438        ahash = &p->derived.alg;
1439        ahash->digest = n2_hmac_async_digest;
1440        ahash->setkey = n2_hmac_async_setkey;
1441
1442        base = &ahash->halg.base;
1443        snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
1444        snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
1445
1446        base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
1447        base->cra_init = n2_hmac_cra_init;
1448        base->cra_exit = n2_hmac_cra_exit;
1449
1450        list_add(&p->derived.entry, &hmac_algs);
1451        err = crypto_register_ahash(ahash);
1452        if (err) {
1453                pr_err("%s alg registration failed\n", base->cra_name);
1454                list_del(&p->derived.entry);
1455                kfree(p);
1456        } else {
1457                pr_info("%s alg registered\n", base->cra_name);
1458        }
1459        return err;
1460}
1461
1462static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
1463{
1464        struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1465        struct hash_alg_common *halg;
1466        struct crypto_alg *base;
1467        struct ahash_alg *ahash;
1468        int err;
1469
1470        if (!p)
1471                return -ENOMEM;
1472
1473        p->hash_zero = tmpl->hash_zero;
1474        p->hash_init = tmpl->hash_init;
1475        p->auth_type = tmpl->auth_type;
1476        p->hmac_type = tmpl->hmac_type;
1477        p->hw_op_hashsz = tmpl->hw_op_hashsz;
1478        p->digest_size = tmpl->digest_size;
1479
1480        ahash = &p->alg;
1481        ahash->init = n2_hash_async_init;
1482        ahash->update = n2_hash_async_update;
1483        ahash->final = n2_hash_async_final;
1484        ahash->finup = n2_hash_async_finup;
1485        ahash->digest = n2_hash_async_digest;
1486
1487        halg = &ahash->halg;
1488        halg->digestsize = tmpl->digest_size;
1489
1490        base = &halg->base;
1491        snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1492        snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1493        base->cra_priority = N2_CRA_PRIORITY;
1494        base->cra_flags = CRYPTO_ALG_TYPE_AHASH |
1495                          CRYPTO_ALG_KERN_DRIVER_ONLY |
1496                          CRYPTO_ALG_NEED_FALLBACK;
1497        base->cra_blocksize = tmpl->block_size;
1498        base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1499        base->cra_module = THIS_MODULE;
1500        base->cra_init = n2_hash_cra_init;
1501        base->cra_exit = n2_hash_cra_exit;
1502
1503        list_add(&p->entry, &ahash_algs);
1504        err = crypto_register_ahash(ahash);
1505        if (err) {
1506                pr_err("%s alg registration failed\n", base->cra_name);
1507                list_del(&p->entry);
1508                kfree(p);
1509        } else {
1510                pr_info("%s alg registered\n", base->cra_name);
1511        }
1512        if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
1513                err = __n2_register_one_hmac(p);
1514        return err;
1515}
1516
1517static int n2_register_algs(void)
1518{
1519        int i, err = 0;
1520
1521        mutex_lock(&spu_lock);
1522        if (algs_registered++)
1523                goto out;
1524
1525        for (i = 0; i < NUM_HASH_TMPLS; i++) {
1526                err = __n2_register_one_ahash(&hash_tmpls[i]);
1527                if (err) {
1528                        __n2_unregister_algs();
1529                        goto out;
1530                }
1531        }
1532        for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1533                err = __n2_register_one_cipher(&cipher_tmpls[i]);
1534                if (err) {
1535                        __n2_unregister_algs();
1536                        goto out;
1537                }
1538        }
1539
1540out:
1541        mutex_unlock(&spu_lock);
1542        return err;
1543}
1544
1545static void n2_unregister_algs(void)
1546{
1547        mutex_lock(&spu_lock);
1548        if (!--algs_registered)
1549                __n2_unregister_algs();
1550        mutex_unlock(&spu_lock);
1551}
1552
1553/* To map CWQ queues to interrupt sources, the hypervisor API provides
1554 * a devino.  This isn't very useful to us because all of the
1555 * interrupts listed in the device_node have been translated to
1556 * Linux virtual IRQ cookie numbers.
1557 *
1558 * So we have to back-translate, going through the 'intr' and 'ino'
1559 * property tables of the n2cp MDESC node, matching it with the OF
1560 * 'interrupts' property entries, in order to to figure out which
1561 * devino goes to which already-translated IRQ.
1562 */
1563static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
1564                             unsigned long dev_ino)
1565{
1566        const unsigned int *dev_intrs;
1567        unsigned int intr;
1568        int i;
1569
1570        for (i = 0; i < ip->num_intrs; i++) {
1571                if (ip->ino_table[i].ino == dev_ino)
1572                        break;
1573        }
1574        if (i == ip->num_intrs)
1575                return -ENODEV;
1576
1577        intr = ip->ino_table[i].intr;
1578
1579        dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
1580        if (!dev_intrs)
1581                return -ENODEV;
1582
1583        for (i = 0; i < dev->archdata.num_irqs; i++) {
1584                if (dev_intrs[i] == intr)
1585                        return i;
1586        }
1587
1588        return -ENODEV;
1589}
1590
1591static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
1592                       const char *irq_name, struct spu_queue *p,
1593                       irq_handler_t handler)
1594{
1595        unsigned long herr;
1596        int index;
1597
1598        herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1599        if (herr)
1600                return -EINVAL;
1601
1602        index = find_devino_index(dev, ip, p->devino);
1603        if (index < 0)
1604                return index;
1605
1606        p->irq = dev->archdata.irqs[index];
1607
1608        sprintf(p->irq_name, "%s-%d", irq_name, index);
1609
1610        return request_irq(p->irq, handler, 0, p->irq_name, p);
1611}
1612
1613static struct kmem_cache *queue_cache[2];
1614
1615static void *new_queue(unsigned long q_type)
1616{
1617        return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1618}
1619
1620static void free_queue(void *p, unsigned long q_type)
1621{
1622        return kmem_cache_free(queue_cache[q_type - 1], p);
1623}
1624
1625static int queue_cache_init(void)
1626{
1627        if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1628                queue_cache[HV_NCS_QTYPE_MAU - 1] =
1629                        kmem_cache_create("mau_queue",
1630                                          (MAU_NUM_ENTRIES *
1631                                           MAU_ENTRY_SIZE),
1632                                          MAU_ENTRY_SIZE, 0, NULL);
1633        if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1634                return -ENOMEM;
1635
1636        if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1637                queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1638                        kmem_cache_create("cwq_queue",
1639                                          (CWQ_NUM_ENTRIES *
1640                                           CWQ_ENTRY_SIZE),
1641                                          CWQ_ENTRY_SIZE, 0, NULL);
1642        if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1643                kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1644                return -ENOMEM;
1645        }
1646        return 0;
1647}
1648
1649static void queue_cache_destroy(void)
1650{
1651        kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1652        kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1653}
1654
1655static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1656{
1657        cpumask_var_t old_allowed;
1658        unsigned long hv_ret;
1659
1660        if (cpumask_empty(&p->sharing))
1661                return -EINVAL;
1662
1663        if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1664                return -ENOMEM;
1665
1666        cpumask_copy(old_allowed, &current->cpus_allowed);
1667
1668        set_cpus_allowed_ptr(current, &p->sharing);
1669
1670        hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1671                                 CWQ_NUM_ENTRIES, &p->qhandle);
1672        if (!hv_ret)
1673                sun4v_ncs_sethead_marker(p->qhandle, 0);
1674
1675        set_cpus_allowed_ptr(current, old_allowed);
1676
1677        free_cpumask_var(old_allowed);
1678
1679        return (hv_ret ? -EINVAL : 0);
1680}
1681
1682static int spu_queue_setup(struct spu_queue *p)
1683{
1684        int err;
1685
1686        p->q = new_queue(p->q_type);
1687        if (!p->q)
1688                return -ENOMEM;
1689
1690        err = spu_queue_register(p, p->q_type);
1691        if (err) {
1692                free_queue(p->q, p->q_type);
1693                p->q = NULL;
1694        }
1695
1696        return err;
1697}
1698
1699static void spu_queue_destroy(struct spu_queue *p)
1700{
1701        unsigned long hv_ret;
1702
1703        if (!p->q)
1704                return;
1705
1706        hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1707
1708        if (!hv_ret)
1709                free_queue(p->q, p->q_type);
1710}
1711
1712static void spu_list_destroy(struct list_head *list)
1713{
1714        struct spu_queue *p, *n;
1715
1716        list_for_each_entry_safe(p, n, list, list) {
1717                int i;
1718
1719                for (i = 0; i < NR_CPUS; i++) {
1720                        if (cpu_to_cwq[i] == p)
1721                                cpu_to_cwq[i] = NULL;
1722                }
1723
1724                if (p->irq) {
1725                        free_irq(p->irq, p);
1726                        p->irq = 0;
1727                }
1728                spu_queue_destroy(p);
1729                list_del(&p->list);
1730                kfree(p);
1731        }
1732}
1733
1734/* Walk the backward arcs of a CWQ 'exec-unit' node,
1735 * gathering cpu membership information.
1736 */
1737static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1738                               struct platform_device *dev,
1739                               u64 node, struct spu_queue *p,
1740                               struct spu_queue **table)
1741{
1742        u64 arc;
1743
1744        mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1745                u64 tgt = mdesc_arc_target(mdesc, arc);
1746                const char *name = mdesc_node_name(mdesc, tgt);
1747                const u64 *id;
1748
1749                if (strcmp(name, "cpu"))
1750                        continue;
1751                id = mdesc_get_property(mdesc, tgt, "id", NULL);
1752                if (table[*id] != NULL) {
1753                        dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
1754                                dev->dev.of_node->full_name);
1755                        return -EINVAL;
1756                }
1757                cpu_set(*id, p->sharing);
1758                table[*id] = p;
1759        }
1760        return 0;
1761}
1762
1763/* Process an 'exec-unit' MDESC node of type 'cwq'.  */
1764static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1765                            struct platform_device *dev, struct mdesc_handle *mdesc,
1766                            u64 node, const char *iname, unsigned long q_type,
1767                            irq_handler_t handler, struct spu_queue **table)
1768{
1769        struct spu_queue *p;
1770        int err;
1771
1772        p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1773        if (!p) {
1774                dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
1775                        dev->dev.of_node->full_name);
1776                return -ENOMEM;
1777        }
1778
1779        cpus_clear(p->sharing);
1780        spin_lock_init(&p->lock);
1781        p->q_type = q_type;
1782        INIT_LIST_HEAD(&p->jobs);
1783        list_add(&p->list, list);
1784
1785        err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1786        if (err)
1787                return err;
1788
1789        err = spu_queue_setup(p);
1790        if (err)
1791                return err;
1792
1793        return spu_map_ino(dev, ip, iname, p, handler);
1794}
1795
1796static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
1797                          struct spu_mdesc_info *ip, struct list_head *list,
1798                          const char *exec_name, unsigned long q_type,
1799                          irq_handler_t handler, struct spu_queue **table)
1800{
1801        int err = 0;
1802        u64 node;
1803
1804        mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1805                const char *type;
1806
1807                type = mdesc_get_property(mdesc, node, "type", NULL);
1808                if (!type || strcmp(type, exec_name))
1809                        continue;
1810
1811                err = handle_exec_unit(ip, list, dev, mdesc, node,
1812                                       exec_name, q_type, handler, table);
1813                if (err) {
1814                        spu_list_destroy(list);
1815                        break;
1816                }
1817        }
1818
1819        return err;
1820}
1821
1822static int get_irq_props(struct mdesc_handle *mdesc, u64 node,
1823                         struct spu_mdesc_info *ip)
1824{
1825        const u64 *ino;
1826        int ino_len;
1827        int i;
1828
1829        ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1830        if (!ino) {
1831                printk("NO 'ino'\n");
1832                return -ENODEV;
1833        }
1834
1835        ip->num_intrs = ino_len / sizeof(u64);
1836        ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1837                                 ip->num_intrs),
1838                                GFP_KERNEL);
1839        if (!ip->ino_table)
1840                return -ENOMEM;
1841
1842        for (i = 0; i < ip->num_intrs; i++) {
1843                struct ino_blob *b = &ip->ino_table[i];
1844                b->intr = i + 1;
1845                b->ino = ino[i];
1846        }
1847
1848        return 0;
1849}
1850
1851static int grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1852                                struct platform_device *dev,
1853                                struct spu_mdesc_info *ip,
1854                                const char *node_name)
1855{
1856        const unsigned int *reg;
1857        u64 node;
1858
1859        reg = of_get_property(dev->dev.of_node, "reg", NULL);
1860        if (!reg)
1861                return -ENODEV;
1862
1863        mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1864                const char *name;
1865                const u64 *chdl;
1866
1867                name = mdesc_get_property(mdesc, node, "name", NULL);
1868                if (!name || strcmp(name, node_name))
1869                        continue;
1870                chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1871                if (!chdl || (*chdl != *reg))
1872                        continue;
1873                ip->cfg_handle = *chdl;
1874                return get_irq_props(mdesc, node, ip);
1875        }
1876
1877        return -ENODEV;
1878}
1879
1880static unsigned long n2_spu_hvapi_major;
1881static unsigned long n2_spu_hvapi_minor;
1882
1883static int n2_spu_hvapi_register(void)
1884{
1885        int err;
1886
1887        n2_spu_hvapi_major = 2;
1888        n2_spu_hvapi_minor = 0;
1889
1890        err = sun4v_hvapi_register(HV_GRP_NCS,
1891                                   n2_spu_hvapi_major,
1892                                   &n2_spu_hvapi_minor);
1893
1894        if (!err)
1895                pr_info("Registered NCS HVAPI version %lu.%lu\n",
1896                        n2_spu_hvapi_major,
1897                        n2_spu_hvapi_minor);
1898
1899        return err;
1900}
1901
1902static void n2_spu_hvapi_unregister(void)
1903{
1904        sun4v_hvapi_unregister(HV_GRP_NCS);
1905}
1906
1907static int global_ref;
1908
1909static int grab_global_resources(void)
1910{
1911        int err = 0;
1912
1913        mutex_lock(&spu_lock);
1914
1915        if (global_ref++)
1916                goto out;
1917
1918        err = n2_spu_hvapi_register();
1919        if (err)
1920                goto out;
1921
1922        err = queue_cache_init();
1923        if (err)
1924                goto out_hvapi_release;
1925
1926        err = -ENOMEM;
1927        cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1928                             GFP_KERNEL);
1929        if (!cpu_to_cwq)
1930                goto out_queue_cache_destroy;
1931
1932        cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1933                             GFP_KERNEL);
1934        if (!cpu_to_mau)
1935                goto out_free_cwq_table;
1936
1937        err = 0;
1938
1939out:
1940        if (err)
1941                global_ref--;
1942        mutex_unlock(&spu_lock);
1943        return err;
1944
1945out_free_cwq_table:
1946        kfree(cpu_to_cwq);
1947        cpu_to_cwq = NULL;
1948
1949out_queue_cache_destroy:
1950        queue_cache_destroy();
1951
1952out_hvapi_release:
1953        n2_spu_hvapi_unregister();
1954        goto out;
1955}
1956
1957static void release_global_resources(void)
1958{
1959        mutex_lock(&spu_lock);
1960        if (!--global_ref) {
1961                kfree(cpu_to_cwq);
1962                cpu_to_cwq = NULL;
1963
1964                kfree(cpu_to_mau);
1965                cpu_to_mau = NULL;
1966
1967                queue_cache_destroy();
1968                n2_spu_hvapi_unregister();
1969        }
1970        mutex_unlock(&spu_lock);
1971}
1972
1973static struct n2_crypto *alloc_n2cp(void)
1974{
1975        struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1976
1977        if (np)
1978                INIT_LIST_HEAD(&np->cwq_list);
1979
1980        return np;
1981}
1982
1983static void free_n2cp(struct n2_crypto *np)
1984{
1985        if (np->cwq_info.ino_table) {
1986                kfree(np->cwq_info.ino_table);
1987                np->cwq_info.ino_table = NULL;
1988        }
1989
1990        kfree(np);
1991}
1992
1993static void n2_spu_driver_version(void)
1994{
1995        static int n2_spu_version_printed;
1996
1997        if (n2_spu_version_printed++ == 0)
1998                pr_info("%s", version);
1999}
2000
2001static int n2_crypto_probe(struct platform_device *dev)
2002{
2003        struct mdesc_handle *mdesc;
2004        const char *full_name;
2005        struct n2_crypto *np;
2006        int err;
2007
2008        n2_spu_driver_version();
2009
2010        full_name = dev->dev.of_node->full_name;
2011        pr_info("Found N2CP at %s\n", full_name);
2012
2013        np = alloc_n2cp();
2014        if (!np) {
2015                dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
2016                        full_name);
2017                return -ENOMEM;
2018        }
2019
2020        err = grab_global_resources();
2021        if (err) {
2022                dev_err(&dev->dev, "%s: Unable to grab "
2023                        "global resources.\n", full_name);
2024                goto out_free_n2cp;
2025        }
2026
2027        mdesc = mdesc_grab();
2028
2029        if (!mdesc) {
2030                dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2031                        full_name);
2032                err = -ENODEV;
2033                goto out_free_global;
2034        }
2035        err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
2036        if (err) {
2037                dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2038                        full_name);
2039                mdesc_release(mdesc);
2040                goto out_free_global;
2041        }
2042
2043        err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
2044                             "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
2045                             cpu_to_cwq);
2046        mdesc_release(mdesc);
2047
2048        if (err) {
2049                dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
2050                        full_name);
2051                goto out_free_global;
2052        }
2053
2054        err = n2_register_algs();
2055        if (err) {
2056                dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
2057                        full_name);
2058                goto out_free_spu_list;
2059        }
2060
2061        dev_set_drvdata(&dev->dev, np);
2062
2063        return 0;
2064
2065out_free_spu_list:
2066        spu_list_destroy(&np->cwq_list);
2067
2068out_free_global:
2069        release_global_resources();
2070
2071out_free_n2cp:
2072        free_n2cp(np);
2073
2074        return err;
2075}
2076
2077static int n2_crypto_remove(struct platform_device *dev)
2078{
2079        struct n2_crypto *np = dev_get_drvdata(&dev->dev);
2080
2081        n2_unregister_algs();
2082
2083        spu_list_destroy(&np->cwq_list);
2084
2085        release_global_resources();
2086
2087        free_n2cp(np);
2088
2089        return 0;
2090}
2091
2092static struct n2_mau *alloc_ncp(void)
2093{
2094        struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
2095
2096        if (mp)
2097                INIT_LIST_HEAD(&mp->mau_list);
2098
2099        return mp;
2100}
2101
2102static void free_ncp(struct n2_mau *mp)
2103{
2104        if (mp->mau_info.ino_table) {
2105                kfree(mp->mau_info.ino_table);
2106                mp->mau_info.ino_table = NULL;
2107        }
2108
2109        kfree(mp);
2110}
2111
2112static int n2_mau_probe(struct platform_device *dev)
2113{
2114        struct mdesc_handle *mdesc;
2115        const char *full_name;
2116        struct n2_mau *mp;
2117        int err;
2118
2119        n2_spu_driver_version();
2120
2121        full_name = dev->dev.of_node->full_name;
2122        pr_info("Found NCP at %s\n", full_name);
2123
2124        mp = alloc_ncp();
2125        if (!mp) {
2126                dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
2127                        full_name);
2128                return -ENOMEM;
2129        }
2130
2131        err = grab_global_resources();
2132        if (err) {
2133                dev_err(&dev->dev, "%s: Unable to grab "
2134                        "global resources.\n", full_name);
2135                goto out_free_ncp;
2136        }
2137
2138        mdesc = mdesc_grab();
2139
2140        if (!mdesc) {
2141                dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2142                        full_name);
2143                err = -ENODEV;
2144                goto out_free_global;
2145        }
2146
2147        err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
2148        if (err) {
2149                dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2150                        full_name);
2151                mdesc_release(mdesc);
2152                goto out_free_global;
2153        }
2154
2155        err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
2156                             "mau", HV_NCS_QTYPE_MAU, mau_intr,
2157                             cpu_to_mau);
2158        mdesc_release(mdesc);
2159
2160        if (err) {
2161                dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
2162                        full_name);
2163                goto out_free_global;
2164        }
2165
2166        dev_set_drvdata(&dev->dev, mp);
2167
2168        return 0;
2169
2170out_free_global:
2171        release_global_resources();
2172
2173out_free_ncp:
2174        free_ncp(mp);
2175
2176        return err;
2177}
2178
2179static int n2_mau_remove(struct platform_device *dev)
2180{
2181        struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2182
2183        spu_list_destroy(&mp->mau_list);
2184
2185        release_global_resources();
2186
2187        free_ncp(mp);
2188
2189        return 0;
2190}
2191
2192static struct of_device_id n2_crypto_match[] = {
2193        {
2194                .name = "n2cp",
2195                .compatible = "SUNW,n2-cwq",
2196        },
2197        {
2198                .name = "n2cp",
2199                .compatible = "SUNW,vf-cwq",
2200        },
2201        {
2202                .name = "n2cp",
2203                .compatible = "SUNW,kt-cwq",
2204        },
2205        {},
2206};
2207
2208MODULE_DEVICE_TABLE(of, n2_crypto_match);
2209
2210static struct platform_driver n2_crypto_driver = {
2211        .driver = {
2212                .name           =       "n2cp",
2213                .of_match_table =       n2_crypto_match,
2214        },
2215        .probe          =       n2_crypto_probe,
2216        .remove         =       n2_crypto_remove,
2217};
2218
2219static struct of_device_id n2_mau_match[] = {
2220        {
2221                .name = "ncp",
2222                .compatible = "SUNW,n2-mau",
2223        },
2224        {
2225                .name = "ncp",
2226                .compatible = "SUNW,vf-mau",
2227        },
2228        {
2229                .name = "ncp",
2230                .compatible = "SUNW,kt-mau",
2231        },
2232        {},
2233};
2234
2235MODULE_DEVICE_TABLE(of, n2_mau_match);
2236
2237static struct platform_driver n2_mau_driver = {
2238        .driver = {
2239                .name           =       "ncp",
2240                .of_match_table =       n2_mau_match,
2241        },
2242        .probe          =       n2_mau_probe,
2243        .remove         =       n2_mau_remove,
2244};
2245
2246static int __init n2_init(void)
2247{
2248        int err = platform_driver_register(&n2_crypto_driver);
2249
2250        if (!err) {
2251                err = platform_driver_register(&n2_mau_driver);
2252                if (err)
2253                        platform_driver_unregister(&n2_crypto_driver);
2254        }
2255        return err;
2256}
2257
2258static void __exit n2_exit(void)
2259{
2260        platform_driver_unregister(&n2_mau_driver);
2261        platform_driver_unregister(&n2_crypto_driver);
2262}
2263
2264module_init(n2_init);
2265module_exit(n2_exit);
2266
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.