linux/drivers/crypto/n2_core.c
<<
>>
Prefs
   1/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
   2 *
   3 * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
   4 */
   5
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8#include <linux/kernel.h>
   9#include <linux/module.h>
  10#include <linux/of.h>
  11#include <linux/of_device.h>
  12#include <linux/cpumask.h>
  13#include <linux/slab.h>
  14#include <linux/interrupt.h>
  15#include <linux/crypto.h>
  16#include <crypto/md5.h>
  17#include <crypto/sha.h>
  18#include <crypto/aes.h>
  19#include <crypto/des.h>
  20#include <linux/mutex.h>
  21#include <linux/delay.h>
  22#include <linux/sched.h>
  23
  24#include <crypto/internal/hash.h>
  25#include <crypto/scatterwalk.h>
  26#include <crypto/algapi.h>
  27
  28#include <asm/hypervisor.h>
  29#include <asm/mdesc.h>
  30
  31#include "n2_core.h"
  32
  33#define DRV_MODULE_NAME         "n2_crypto"
  34#define DRV_MODULE_VERSION      "0.2"
  35#define DRV_MODULE_RELDATE      "July 28, 2011"
  36
  37static char version[] __devinitdata =
  38        DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  39
  40MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
  41MODULE_DESCRIPTION("Niagara2 Crypto driver");
  42MODULE_LICENSE("GPL");
  43MODULE_VERSION(DRV_MODULE_VERSION);
  44
  45#define N2_CRA_PRIORITY         200
  46
  47static DEFINE_MUTEX(spu_lock);
  48
  49struct spu_queue {
  50        cpumask_t               sharing;
  51        unsigned long           qhandle;
  52
  53        spinlock_t              lock;
  54        u8                      q_type;
  55        void                    *q;
  56        unsigned long           head;
  57        unsigned long           tail;
  58        struct list_head        jobs;
  59
  60        unsigned long           devino;
  61
  62        char                    irq_name[32];
  63        unsigned int            irq;
  64
  65        struct list_head        list;
  66};
  67
  68static struct spu_queue **cpu_to_cwq;
  69static struct spu_queue **cpu_to_mau;
  70
  71static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
  72{
  73        if (q->q_type == HV_NCS_QTYPE_MAU) {
  74                off += MAU_ENTRY_SIZE;
  75                if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
  76                        off = 0;
  77        } else {
  78                off += CWQ_ENTRY_SIZE;
  79                if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
  80                        off = 0;
  81        }
  82        return off;
  83}
  84
  85struct n2_request_common {
  86        struct list_head        entry;
  87        unsigned int            offset;
  88};
  89#define OFFSET_NOT_RUNNING      (~(unsigned int)0)
  90
  91/* An async job request records the final tail value it used in
  92 * n2_request_common->offset, test to see if that offset is in
  93 * the range old_head, new_head, inclusive.
  94 */
  95static inline bool job_finished(struct spu_queue *q, unsigned int offset,
  96                                unsigned long old_head, unsigned long new_head)
  97{
  98        if (old_head <= new_head) {
  99                if (offset > old_head && offset <= new_head)
 100                        return true;
 101        } else {
 102                if (offset > old_head || offset <= new_head)
 103                        return true;
 104        }
 105        return false;
 106}
 107
 108/* When the HEAD marker is unequal to the actual HEAD, we get
 109 * a virtual device INO interrupt.  We should process the
 110 * completed CWQ entries and adjust the HEAD marker to clear
 111 * the IRQ.
 112 */
 113static irqreturn_t cwq_intr(int irq, void *dev_id)
 114{
 115        unsigned long off, new_head, hv_ret;
 116        struct spu_queue *q = dev_id;
 117
 118        pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
 119               smp_processor_id(), q->qhandle);
 120
 121        spin_lock(&q->lock);
 122
 123        hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
 124
 125        pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
 126               smp_processor_id(), new_head, hv_ret);
 127
 128        for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
 129                /* XXX ... XXX */
 130        }
 131
 132        hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
 133        if (hv_ret == HV_EOK)
 134                q->head = new_head;
 135
 136        spin_unlock(&q->lock);
 137
 138        return IRQ_HANDLED;
 139}
 140
 141static irqreturn_t mau_intr(int irq, void *dev_id)
 142{
 143        struct spu_queue *q = dev_id;
 144        unsigned long head, hv_ret;
 145
 146        spin_lock(&q->lock);
 147
 148        pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
 149               smp_processor_id(), q->qhandle);
 150
 151        hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
 152
 153        pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
 154               smp_processor_id(), head, hv_ret);
 155
 156        sun4v_ncs_sethead_marker(q->qhandle, head);
 157
 158        spin_unlock(&q->lock);
 159
 160        return IRQ_HANDLED;
 161}
 162
 163static void *spu_queue_next(struct spu_queue *q, void *cur)
 164{
 165        return q->q + spu_next_offset(q, cur - q->q);
 166}
 167
 168static int spu_queue_num_free(struct spu_queue *q)
 169{
 170        unsigned long head = q->head;
 171        unsigned long tail = q->tail;
 172        unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
 173        unsigned long diff;
 174
 175        if (head > tail)
 176                diff = head - tail;
 177        else
 178                diff = (end - tail) + head;
 179
 180        return (diff / CWQ_ENTRY_SIZE) - 1;
 181}
 182
 183static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
 184{
 185        int avail = spu_queue_num_free(q);
 186
 187        if (avail >= num_entries)
 188                return q->q + q->tail;
 189
 190        return NULL;
 191}
 192
 193static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
 194{
 195        unsigned long hv_ret, new_tail;
 196
 197        new_tail = spu_next_offset(q, last - q->q);
 198
 199        hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
 200        if (hv_ret == HV_EOK)
 201                q->tail = new_tail;
 202        return hv_ret;
 203}
 204
 205static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
 206                             int enc_type, int auth_type,
 207                             unsigned int hash_len,
 208                             bool sfas, bool sob, bool eob, bool encrypt,
 209                             int opcode)
 210{
 211        u64 word = (len - 1) & CONTROL_LEN;
 212
 213        word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
 214        word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
 215        word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
 216        if (sfas)
 217                word |= CONTROL_STORE_FINAL_AUTH_STATE;
 218        if (sob)
 219                word |= CONTROL_START_OF_BLOCK;
 220        if (eob)
 221                word |= CONTROL_END_OF_BLOCK;
 222        if (encrypt)
 223                word |= CONTROL_ENCRYPT;
 224        if (hmac_key_len)
 225                word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
 226        if (hash_len)
 227                word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
 228
 229        return word;
 230}
 231
 232#if 0
 233static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
 234{
 235        if (this_len >= 64 ||
 236            qp->head != qp->tail)
 237                return true;
 238        return false;
 239}
 240#endif
 241
 242struct n2_ahash_alg {
 243        struct list_head        entry;
 244        const char              *hash_zero;
 245        const u32               *hash_init;
 246        u8                      hw_op_hashsz;
 247        u8                      digest_size;
 248        u8                      auth_type;
 249        u8                      hmac_type;
 250        struct ahash_alg        alg;
 251};
 252
 253static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
 254{
 255        struct crypto_alg *alg = tfm->__crt_alg;
 256        struct ahash_alg *ahash_alg;
 257
 258        ahash_alg = container_of(alg, struct ahash_alg, halg.base);
 259
 260        return container_of(ahash_alg, struct n2_ahash_alg, alg);
 261}
 262
 263struct n2_hmac_alg {
 264        const char              *child_alg;
 265        struct n2_ahash_alg     derived;
 266};
 267
 268static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
 269{
 270        struct crypto_alg *alg = tfm->__crt_alg;
 271        struct ahash_alg *ahash_alg;
 272
 273        ahash_alg = container_of(alg, struct ahash_alg, halg.base);
 274
 275        return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
 276}
 277
 278struct n2_hash_ctx {
 279        struct crypto_ahash             *fallback_tfm;
 280};
 281
 282#define N2_HASH_KEY_MAX                 32 /* HW limit for all HMAC requests */
 283
 284struct n2_hmac_ctx {
 285        struct n2_hash_ctx              base;
 286
 287        struct crypto_shash             *child_shash;
 288
 289        int                             hash_key_len;
 290        unsigned char                   hash_key[N2_HASH_KEY_MAX];
 291};
 292
 293struct n2_hash_req_ctx {
 294        union {
 295                struct md5_state        md5;
 296                struct sha1_state       sha1;
 297                struct sha256_state     sha256;
 298        } u;
 299
 300        struct ahash_request            fallback_req;
 301};
 302
 303static int n2_hash_async_init(struct ahash_request *req)
 304{
 305        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 306        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 307        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 308
 309        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 310        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 311
 312        return crypto_ahash_init(&rctx->fallback_req);
 313}
 314
 315static int n2_hash_async_update(struct ahash_request *req)
 316{
 317        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 318        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 319        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 320
 321        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 322        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 323        rctx->fallback_req.nbytes = req->nbytes;
 324        rctx->fallback_req.src = req->src;
 325
 326        return crypto_ahash_update(&rctx->fallback_req);
 327}
 328
 329static int n2_hash_async_final(struct ahash_request *req)
 330{
 331        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 332        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 333        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 334
 335        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 336        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 337        rctx->fallback_req.result = req->result;
 338
 339        return crypto_ahash_final(&rctx->fallback_req);
 340}
 341
 342static int n2_hash_async_finup(struct ahash_request *req)
 343{
 344        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 345        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 346        struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 347
 348        ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 349        rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 350        rctx->fallback_req.nbytes = req->nbytes;
 351        rctx->fallback_req.src = req->src;
 352        rctx->fallback_req.result = req->result;
 353
 354        return crypto_ahash_finup(&rctx->fallback_req);
 355}
 356
 357static int n2_hash_cra_init(struct crypto_tfm *tfm)
 358{
 359        const char *fallback_driver_name = tfm->__crt_alg->cra_name;
 360        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 361        struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 362        struct crypto_ahash *fallback_tfm;
 363        int err;
 364
 365        fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 366                                          CRYPTO_ALG_NEED_FALLBACK);
 367        if (IS_ERR(fallback_tfm)) {
 368                pr_warning("Fallback driver '%s' could not be loaded!\n",
 369                           fallback_driver_name);
 370                err = PTR_ERR(fallback_tfm);
 371                goto out;
 372        }
 373
 374        crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 375                                         crypto_ahash_reqsize(fallback_tfm)));
 376
 377        ctx->fallback_tfm = fallback_tfm;
 378        return 0;
 379
 380out:
 381        return err;
 382}
 383
 384static void n2_hash_cra_exit(struct crypto_tfm *tfm)
 385{
 386        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 387        struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 388
 389        crypto_free_ahash(ctx->fallback_tfm);
 390}
 391
 392static int n2_hmac_cra_init(struct crypto_tfm *tfm)
 393{
 394        const char *fallback_driver_name = tfm->__crt_alg->cra_name;
 395        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 396        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
 397        struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
 398        struct crypto_ahash *fallback_tfm;
 399        struct crypto_shash *child_shash;
 400        int err;
 401
 402        fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
 403                                          CRYPTO_ALG_NEED_FALLBACK);
 404        if (IS_ERR(fallback_tfm)) {
 405                pr_warning("Fallback driver '%s' could not be loaded!\n",
 406                           fallback_driver_name);
 407                err = PTR_ERR(fallback_tfm);
 408                goto out;
 409        }
 410
 411        child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
 412        if (IS_ERR(child_shash)) {
 413                pr_warning("Child shash '%s' could not be loaded!\n",
 414                           n2alg->child_alg);
 415                err = PTR_ERR(child_shash);
 416                goto out_free_fallback;
 417        }
 418
 419        crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
 420                                         crypto_ahash_reqsize(fallback_tfm)));
 421
 422        ctx->child_shash = child_shash;
 423        ctx->base.fallback_tfm = fallback_tfm;
 424        return 0;
 425
 426out_free_fallback:
 427        crypto_free_ahash(fallback_tfm);
 428
 429out:
 430        return err;
 431}
 432
 433static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
 434{
 435        struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
 436        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
 437
 438        crypto_free_ahash(ctx->base.fallback_tfm);
 439        crypto_free_shash(ctx->child_shash);
 440}
 441
 442static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
 443                                unsigned int keylen)
 444{
 445        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
 446        struct crypto_shash *child_shash = ctx->child_shash;
 447        struct crypto_ahash *fallback_tfm;
 448        struct {
 449                struct shash_desc shash;
 450                char ctx[crypto_shash_descsize(child_shash)];
 451        } desc;
 452        int err, bs, ds;
 453
 454        fallback_tfm = ctx->base.fallback_tfm;
 455        err = crypto_ahash_setkey(fallback_tfm, key, keylen);
 456        if (err)
 457                return err;
 458
 459        desc.shash.tfm = child_shash;
 460        desc.shash.flags = crypto_ahash_get_flags(tfm) &
 461                CRYPTO_TFM_REQ_MAY_SLEEP;
 462
 463        bs = crypto_shash_blocksize(child_shash);
 464        ds = crypto_shash_digestsize(child_shash);
 465        BUG_ON(ds > N2_HASH_KEY_MAX);
 466        if (keylen > bs) {
 467                err = crypto_shash_digest(&desc.shash, key, keylen,
 468                                          ctx->hash_key);
 469                if (err)
 470                        return err;
 471                keylen = ds;
 472        } else if (keylen <= N2_HASH_KEY_MAX)
 473                memcpy(ctx->hash_key, key, keylen);
 474
 475        ctx->hash_key_len = keylen;
 476
 477        return err;
 478}
 479
 480static unsigned long wait_for_tail(struct spu_queue *qp)
 481{
 482        unsigned long head, hv_ret;
 483
 484        do {
 485                hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
 486                if (hv_ret != HV_EOK) {
 487                        pr_err("Hypervisor error on gethead\n");
 488                        break;
 489                }
 490                if (head == qp->tail) {
 491                        qp->head = head;
 492                        break;
 493                }
 494        } while (1);
 495        return hv_ret;
 496}
 497
 498static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
 499                                              struct cwq_initial_entry *ent)
 500{
 501        unsigned long hv_ret = spu_queue_submit(qp, ent);
 502
 503        if (hv_ret == HV_EOK)
 504                hv_ret = wait_for_tail(qp);
 505
 506        return hv_ret;
 507}
 508
 509static int n2_do_async_digest(struct ahash_request *req,
 510                              unsigned int auth_type, unsigned int digest_size,
 511                              unsigned int result_size, void *hash_loc,
 512                              unsigned long auth_key, unsigned int auth_key_len)
 513{
 514        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 515        struct cwq_initial_entry *ent;
 516        struct crypto_hash_walk walk;
 517        struct spu_queue *qp;
 518        unsigned long flags;
 519        int err = -ENODEV;
 520        int nbytes, cpu;
 521
 522        /* The total effective length of the operation may not
 523         * exceed 2^16.
 524         */
 525        if (unlikely(req->nbytes > (1 << 16))) {
 526                struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 527                struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 528
 529                ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 530                rctx->fallback_req.base.flags =
 531                        req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 532                rctx->fallback_req.nbytes = req->nbytes;
 533                rctx->fallback_req.src = req->src;
 534                rctx->fallback_req.result = req->result;
 535
 536                return crypto_ahash_digest(&rctx->fallback_req);
 537        }
 538
 539        nbytes = crypto_hash_walk_first(req, &walk);
 540
 541        cpu = get_cpu();
 542        qp = cpu_to_cwq[cpu];
 543        if (!qp)
 544                goto out;
 545
 546        spin_lock_irqsave(&qp->lock, flags);
 547
 548        /* XXX can do better, improve this later by doing a by-hand scatterlist
 549         * XXX walk, etc.
 550         */
 551        ent = qp->q + qp->tail;
 552
 553        ent->control = control_word_base(nbytes, auth_key_len, 0,
 554                                         auth_type, digest_size,
 555                                         false, true, false, false,
 556                                         OPCODE_INPLACE_BIT |
 557                                         OPCODE_AUTH_MAC);
 558        ent->src_addr = __pa(walk.data);
 559        ent->auth_key_addr = auth_key;
 560        ent->auth_iv_addr = __pa(hash_loc);
 561        ent->final_auth_state_addr = 0UL;
 562        ent->enc_key_addr = 0UL;
 563        ent->enc_iv_addr = 0UL;
 564        ent->dest_addr = __pa(hash_loc);
 565
 566        nbytes = crypto_hash_walk_done(&walk, 0);
 567        while (nbytes > 0) {
 568                ent = spu_queue_next(qp, ent);
 569
 570                ent->control = (nbytes - 1);
 571                ent->src_addr = __pa(walk.data);
 572                ent->auth_key_addr = 0UL;
 573                ent->auth_iv_addr = 0UL;
 574                ent->final_auth_state_addr = 0UL;
 575                ent->enc_key_addr = 0UL;
 576                ent->enc_iv_addr = 0UL;
 577                ent->dest_addr = 0UL;
 578
 579                nbytes = crypto_hash_walk_done(&walk, 0);
 580        }
 581        ent->control |= CONTROL_END_OF_BLOCK;
 582
 583        if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
 584                err = -EINVAL;
 585        else
 586                err = 0;
 587
 588        spin_unlock_irqrestore(&qp->lock, flags);
 589
 590        if (!err)
 591                memcpy(req->result, hash_loc, result_size);
 592out:
 593        put_cpu();
 594
 595        return err;
 596}
 597
 598static int n2_hash_async_digest(struct ahash_request *req)
 599{
 600        struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
 601        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 602        int ds;
 603
 604        ds = n2alg->digest_size;
 605        if (unlikely(req->nbytes == 0)) {
 606                memcpy(req->result, n2alg->hash_zero, ds);
 607                return 0;
 608        }
 609        memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
 610
 611        return n2_do_async_digest(req, n2alg->auth_type,
 612                                  n2alg->hw_op_hashsz, ds,
 613                                  &rctx->u, 0UL, 0);
 614}
 615
 616static int n2_hmac_async_digest(struct ahash_request *req)
 617{
 618        struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
 619        struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 620        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 621        struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
 622        int ds;
 623
 624        ds = n2alg->derived.digest_size;
 625        if (unlikely(req->nbytes == 0) ||
 626            unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
 627                struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
 628                struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 629
 630                ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 631                rctx->fallback_req.base.flags =
 632                        req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 633                rctx->fallback_req.nbytes = req->nbytes;
 634                rctx->fallback_req.src = req->src;
 635                rctx->fallback_req.result = req->result;
 636
 637                return crypto_ahash_digest(&rctx->fallback_req);
 638        }
 639        memcpy(&rctx->u, n2alg->derived.hash_init,
 640               n2alg->derived.hw_op_hashsz);
 641
 642        return n2_do_async_digest(req, n2alg->derived.hmac_type,
 643                                  n2alg->derived.hw_op_hashsz, ds,
 644                                  &rctx->u,
 645                                  __pa(&ctx->hash_key),
 646                                  ctx->hash_key_len);
 647}
 648
 649struct n2_cipher_context {
 650        int                     key_len;
 651        int                     enc_type;
 652        union {
 653                u8              aes[AES_MAX_KEY_SIZE];
 654                u8              des[DES_KEY_SIZE];
 655                u8              des3[3 * DES_KEY_SIZE];
 656                u8              arc4[258]; /* S-box, X, Y */
 657        } key;
 658};
 659
 660#define N2_CHUNK_ARR_LEN        16
 661
 662struct n2_crypto_chunk {
 663        struct list_head        entry;
 664        unsigned long           iv_paddr : 44;
 665        unsigned long           arr_len : 20;
 666        unsigned long           dest_paddr;
 667        unsigned long           dest_final;
 668        struct {
 669                unsigned long   src_paddr : 44;
 670                unsigned long   src_len : 20;
 671        } arr[N2_CHUNK_ARR_LEN];
 672};
 673
 674struct n2_request_context {
 675        struct ablkcipher_walk  walk;
 676        struct list_head        chunk_list;
 677        struct n2_crypto_chunk  chunk;
 678        u8                      temp_iv[16];
 679};
 680
 681/* The SPU allows some level of flexibility for partial cipher blocks
 682 * being specified in a descriptor.
 683 *
 684 * It merely requires that every descriptor's length field is at least
 685 * as large as the cipher block size.  This means that a cipher block
 686 * can span at most 2 descriptors.  However, this does not allow a
 687 * partial block to span into the final descriptor as that would
 688 * violate the rule (since every descriptor's length must be at lest
 689 * the block size).  So, for example, assuming an 8 byte block size:
 690 *
 691 *      0xe --> 0xa --> 0x8
 692 *
 693 * is a valid length sequence, whereas:
 694 *
 695 *      0xe --> 0xb --> 0x7
 696 *
 697 * is not a valid sequence.
 698 */
 699
 700struct n2_cipher_alg {
 701        struct list_head        entry;
 702        u8                      enc_type;
 703        struct crypto_alg       alg;
 704};
 705
 706static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
 707{
 708        struct crypto_alg *alg = tfm->__crt_alg;
 709
 710        return container_of(alg, struct n2_cipher_alg, alg);
 711}
 712
 713struct n2_cipher_request_context {
 714        struct ablkcipher_walk  walk;
 715};
 716
 717static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 718                         unsigned int keylen)
 719{
 720        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 721        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 722        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 723
 724        ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
 725
 726        switch (keylen) {
 727        case AES_KEYSIZE_128:
 728                ctx->enc_type |= ENC_TYPE_ALG_AES128;
 729                break;
 730        case AES_KEYSIZE_192:
 731                ctx->enc_type |= ENC_TYPE_ALG_AES192;
 732                break;
 733        case AES_KEYSIZE_256:
 734                ctx->enc_type |= ENC_TYPE_ALG_AES256;
 735                break;
 736        default:
 737                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 738                return -EINVAL;
 739        }
 740
 741        ctx->key_len = keylen;
 742        memcpy(ctx->key.aes, key, keylen);
 743        return 0;
 744}
 745
 746static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 747                         unsigned int keylen)
 748{
 749        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 750        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 751        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 752        u32 tmp[DES_EXPKEY_WORDS];
 753        int err;
 754
 755        ctx->enc_type = n2alg->enc_type;
 756
 757        if (keylen != DES_KEY_SIZE) {
 758                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 759                return -EINVAL;
 760        }
 761
 762        err = des_ekey(tmp, key);
 763        if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 764                tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 765                return -EINVAL;
 766        }
 767
 768        ctx->key_len = keylen;
 769        memcpy(ctx->key.des, key, keylen);
 770        return 0;
 771}
 772
 773static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 774                          unsigned int keylen)
 775{
 776        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 777        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 778        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 779
 780        ctx->enc_type = n2alg->enc_type;
 781
 782        if (keylen != (3 * DES_KEY_SIZE)) {
 783                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 784                return -EINVAL;
 785        }
 786        ctx->key_len = keylen;
 787        memcpy(ctx->key.des3, key, keylen);
 788        return 0;
 789}
 790
 791static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
 792                          unsigned int keylen)
 793{
 794        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 795        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 796        struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
 797        u8 *s = ctx->key.arc4;
 798        u8 *x = s + 256;
 799        u8 *y = x + 1;
 800        int i, j, k;
 801
 802        ctx->enc_type = n2alg->enc_type;
 803
 804        j = k = 0;
 805        *x = 0;
 806        *y = 0;
 807        for (i = 0; i < 256; i++)
 808                s[i] = i;
 809        for (i = 0; i < 256; i++) {
 810                u8 a = s[i];
 811                j = (j + key[k] + a) & 0xff;
 812                s[i] = s[j];
 813                s[j] = a;
 814                if (++k >= keylen)
 815                        k = 0;
 816        }
 817
 818        return 0;
 819}
 820
 821static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
 822{
 823        int this_len = nbytes;
 824
 825        this_len -= (nbytes & (block_size - 1));
 826        return this_len > (1 << 16) ? (1 << 16) : this_len;
 827}
 828
 829static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
 830                            struct spu_queue *qp, bool encrypt)
 831{
 832        struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
 833        struct cwq_initial_entry *ent;
 834        bool in_place;
 835        int i;
 836
 837        ent = spu_queue_alloc(qp, cp->arr_len);
 838        if (!ent) {
 839                pr_info("queue_alloc() of %d fails\n",
 840                        cp->arr_len);
 841                return -EBUSY;
 842        }
 843
 844        in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
 845
 846        ent->control = control_word_base(cp->arr[0].src_len,
 847                                         0, ctx->enc_type, 0, 0,
 848                                         false, true, false, encrypt,
 849                                         OPCODE_ENCRYPT |
 850                                         (in_place ? OPCODE_INPLACE_BIT : 0));
 851        ent->src_addr = cp->arr[0].src_paddr;
 852        ent->auth_key_addr = 0UL;
 853        ent->auth_iv_addr = 0UL;
 854        ent->final_auth_state_addr = 0UL;
 855        ent->enc_key_addr = __pa(&ctx->key);
 856        ent->enc_iv_addr = cp->iv_paddr;
 857        ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
 858
 859        for (i = 1; i < cp->arr_len; i++) {
 860                ent = spu_queue_next(qp, ent);
 861
 862                ent->control = cp->arr[i].src_len - 1;
 863                ent->src_addr = cp->arr[i].src_paddr;
 864                ent->auth_key_addr = 0UL;
 865                ent->auth_iv_addr = 0UL;
 866                ent->final_auth_state_addr = 0UL;
 867                ent->enc_key_addr = 0UL;
 868                ent->enc_iv_addr = 0UL;
 869                ent->dest_addr = 0UL;
 870        }
 871        ent->control |= CONTROL_END_OF_BLOCK;
 872
 873        return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
 874}
 875
 876static int n2_compute_chunks(struct ablkcipher_request *req)
 877{
 878        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 879        struct ablkcipher_walk *walk = &rctx->walk;
 880        struct n2_crypto_chunk *chunk;
 881        unsigned long dest_prev;
 882        unsigned int tot_len;
 883        bool prev_in_place;
 884        int err, nbytes;
 885
 886        ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
 887        err = ablkcipher_walk_phys(req, walk);
 888        if (err)
 889                return err;
 890
 891        INIT_LIST_HEAD(&rctx->chunk_list);
 892
 893        chunk = &rctx->chunk;
 894        INIT_LIST_HEAD(&chunk->entry);
 895
 896        chunk->iv_paddr = 0UL;
 897        chunk->arr_len = 0;
 898        chunk->dest_paddr = 0UL;
 899
 900        prev_in_place = false;
 901        dest_prev = ~0UL;
 902        tot_len = 0;
 903
 904        while ((nbytes = walk->nbytes) != 0) {
 905                unsigned long dest_paddr, src_paddr;
 906                bool in_place;
 907                int this_len;
 908
 909                src_paddr = (page_to_phys(walk->src.page) +
 910                             walk->src.offset);
 911                dest_paddr = (page_to_phys(walk->dst.page) +
 912                              walk->dst.offset);
 913                in_place = (src_paddr == dest_paddr);
 914                this_len = cipher_descriptor_len(nbytes, walk->blocksize);
 915
 916                if (chunk->arr_len != 0) {
 917                        if (in_place != prev_in_place ||
 918                            (!prev_in_place &&
 919                             dest_paddr != dest_prev) ||
 920                            chunk->arr_len == N2_CHUNK_ARR_LEN ||
 921                            tot_len + this_len > (1 << 16)) {
 922                                chunk->dest_final = dest_prev;
 923                                list_add_tail(&chunk->entry,
 924                                              &rctx->chunk_list);
 925                                chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
 926                                if (!chunk) {
 927                                        err = -ENOMEM;
 928                                        break;
 929                                }
 930                                INIT_LIST_HEAD(&chunk->entry);
 931                        }
 932                }
 933                if (chunk->arr_len == 0) {
 934                        chunk->dest_paddr = dest_paddr;
 935                        tot_len = 0;
 936                }
 937                chunk->arr[chunk->arr_len].src_paddr = src_paddr;
 938                chunk->arr[chunk->arr_len].src_len = this_len;
 939                chunk->arr_len++;
 940
 941                dest_prev = dest_paddr + this_len;
 942                prev_in_place = in_place;
 943                tot_len += this_len;
 944
 945                err = ablkcipher_walk_done(req, walk, nbytes - this_len);
 946                if (err)
 947                        break;
 948        }
 949        if (!err && chunk->arr_len != 0) {
 950                chunk->dest_final = dest_prev;
 951                list_add_tail(&chunk->entry, &rctx->chunk_list);
 952        }
 953
 954        return err;
 955}
 956
 957static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
 958{
 959        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 960        struct n2_crypto_chunk *c, *tmp;
 961
 962        if (final_iv)
 963                memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
 964
 965        ablkcipher_walk_complete(&rctx->walk);
 966        list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 967                list_del(&c->entry);
 968                if (unlikely(c != &rctx->chunk))
 969                        kfree(c);
 970        }
 971
 972}
 973
 974static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
 975{
 976        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
 977        struct crypto_tfm *tfm = req->base.tfm;
 978        int err = n2_compute_chunks(req);
 979        struct n2_crypto_chunk *c, *tmp;
 980        unsigned long flags, hv_ret;
 981        struct spu_queue *qp;
 982
 983        if (err)
 984                return err;
 985
 986        qp = cpu_to_cwq[get_cpu()];
 987        err = -ENODEV;
 988        if (!qp)
 989                goto out;
 990
 991        spin_lock_irqsave(&qp->lock, flags);
 992
 993        list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
 994                err = __n2_crypt_chunk(tfm, c, qp, encrypt);
 995                if (err)
 996                        break;
 997                list_del(&c->entry);
 998                if (unlikely(c != &rctx->chunk))
 999                        kfree(c);
1000        }
1001        if (!err) {
1002                hv_ret = wait_for_tail(qp);
1003                if (hv_ret != HV_EOK)
1004                        err = -EINVAL;
1005        }
1006
1007        spin_unlock_irqrestore(&qp->lock, flags);
1008
1009out:
1010        put_cpu();
1011
1012        n2_chunk_complete(req, NULL);
1013        return err;
1014}
1015
1016static int n2_encrypt_ecb(struct ablkcipher_request *req)
1017{
1018        return n2_do_ecb(req, true);
1019}
1020
1021static int n2_decrypt_ecb(struct ablkcipher_request *req)
1022{
1023        return n2_do_ecb(req, false);
1024}
1025
1026static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
1027{
1028        struct n2_request_context *rctx = ablkcipher_request_ctx(req);
1029        struct crypto_tfm *tfm = req->base.tfm;
1030        unsigned long flags, hv_ret, iv_paddr;
1031        int err = n2_compute_chunks(req);
1032        struct n2_crypto_chunk *c, *tmp;
1033        struct spu_queue *qp;
1034        void *final_iv_addr;
1035
1036        final_iv_addr = NULL;
1037
1038        if (err)
1039                return err;
1040
1041        qp = cpu_to_cwq[get_cpu()];
1042        err = -ENODEV;
1043        if (!qp)
1044                goto out;
1045
1046        spin_lock_irqsave(&qp->lock, flags);
1047
1048        if (encrypt) {
1049                iv_paddr = __pa(rctx->walk.iv);
1050                list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
1051                                         entry) {
1052                        c->iv_paddr = iv_paddr;
1053                        err = __n2_crypt_chunk(tfm, c, qp, true);
1054                        if (err)
1055                                break;
1056                        iv_paddr = c->dest_final - rctx->walk.blocksize;
1057                        list_del(&c->entry);
1058                        if (unlikely(c != &rctx->chunk))
1059                                kfree(c);
1060                }
1061                final_iv_addr = __va(iv_paddr);
1062        } else {
1063                list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1064                                                 entry) {
1065                        if (c == &rctx->chunk) {
1066                                iv_paddr = __pa(rctx->walk.iv);
1067                        } else {
1068                                iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1069                                            tmp->arr[tmp->arr_len-1].src_len -
1070                                            rctx->walk.blocksize);
1071                        }
1072                        if (!final_iv_addr) {
1073                                unsigned long pa;
1074
1075                                pa = (c->arr[c->arr_len-1].src_paddr +
1076                                      c->arr[c->arr_len-1].src_len -
1077                                      rctx->walk.blocksize);
1078                                final_iv_addr = rctx->temp_iv;
1079                                memcpy(rctx->temp_iv, __va(pa),
1080                                       rctx->walk.blocksize);
1081                        }
1082                        c->iv_paddr = iv_paddr;
1083                        err = __n2_crypt_chunk(tfm, c, qp, false);
1084                        if (err)
1085                                break;
1086                        list_del(&c->entry);
1087                        if (unlikely(c != &rctx->chunk))
1088                                kfree(c);
1089                }
1090        }
1091        if (!err) {
1092                hv_ret = wait_for_tail(qp);
1093                if (hv_ret != HV_EOK)
1094                        err = -EINVAL;
1095        }
1096
1097        spin_unlock_irqrestore(&qp->lock, flags);
1098
1099out:
1100        put_cpu();
1101
1102        n2_chunk_complete(req, err ? NULL : final_iv_addr);
1103        return err;
1104}
1105
1106static int n2_encrypt_chaining(struct ablkcipher_request *req)
1107{
1108        return n2_do_chaining(req, true);
1109}
1110
1111static int n2_decrypt_chaining(struct ablkcipher_request *req)
1112{
1113        return n2_do_chaining(req, false);
1114}
1115
1116struct n2_cipher_tmpl {
1117        const char              *name;
1118        const char              *drv_name;
1119        u8                      block_size;
1120        u8                      enc_type;
1121        struct ablkcipher_alg   ablkcipher;
1122};
1123
1124static const struct n2_cipher_tmpl cipher_tmpls[] = {
1125        /* ARC4: only ECB is supported (chaining bits ignored) */
1126        {       .name           = "ecb(arc4)",
1127                .drv_name       = "ecb-arc4",
1128                .block_size     = 1,
1129                .enc_type       = (ENC_TYPE_ALG_RC4_STREAM |
1130                                   ENC_TYPE_CHAINING_ECB),
1131                .ablkcipher     = {
1132                        .min_keysize    = 1,
1133                        .max_keysize    = 256,
1134                        .setkey         = n2_arc4_setkey,
1135                        .encrypt        = n2_encrypt_ecb,
1136                        .decrypt        = n2_decrypt_ecb,
1137                },
1138        },
1139
1140        /* DES: ECB CBC and CFB are supported */
1141        {       .name           = "ecb(des)",
1142                .drv_name       = "ecb-des",
1143                .block_size     = DES_BLOCK_SIZE,
1144                .enc_type       = (ENC_TYPE_ALG_DES |
1145                                   ENC_TYPE_CHAINING_ECB),
1146                .ablkcipher     = {
1147                        .min_keysize    = DES_KEY_SIZE,
1148                        .max_keysize    = DES_KEY_SIZE,
1149                        .setkey         = n2_des_setkey,
1150                        .encrypt        = n2_encrypt_ecb,
1151                        .decrypt        = n2_decrypt_ecb,
1152                },
1153        },
1154        {       .name           = "cbc(des)",
1155                .drv_name       = "cbc-des",
1156                .block_size     = DES_BLOCK_SIZE,
1157                .enc_type       = (ENC_TYPE_ALG_DES |
1158                                   ENC_TYPE_CHAINING_CBC),
1159                .ablkcipher     = {
1160                        .ivsize         = DES_BLOCK_SIZE,
1161                        .min_keysize    = DES_KEY_SIZE,
1162                        .max_keysize    = DES_KEY_SIZE,
1163                        .setkey         = n2_des_setkey,
1164                        .encrypt        = n2_encrypt_chaining,
1165                        .decrypt        = n2_decrypt_chaining,
1166                },
1167        },
1168        {       .name           = "cfb(des)",
1169                .drv_name       = "cfb-des",
1170                .block_size     = DES_BLOCK_SIZE,
1171                .enc_type       = (ENC_TYPE_ALG_DES |
1172                                   ENC_TYPE_CHAINING_CFB),
1173                .ablkcipher     = {
1174                        .min_keysize    = DES_KEY_SIZE,
1175                        .max_keysize    = DES_KEY_SIZE,
1176                        .setkey         = n2_des_setkey,
1177                        .encrypt        = n2_encrypt_chaining,
1178                        .decrypt        = n2_decrypt_chaining,
1179                },
1180        },
1181
1182        /* 3DES: ECB CBC and CFB are supported */
1183        {       .name           = "ecb(des3_ede)",
1184                .drv_name       = "ecb-3des",
1185                .block_size     = DES_BLOCK_SIZE,
1186                .enc_type       = (ENC_TYPE_ALG_3DES |
1187                                   ENC_TYPE_CHAINING_ECB),
1188                .ablkcipher     = {
1189                        .min_keysize    = 3 * DES_KEY_SIZE,
1190                        .max_keysize    = 3 * DES_KEY_SIZE,
1191                        .setkey         = n2_3des_setkey,
1192                        .encrypt        = n2_encrypt_ecb,
1193                        .decrypt        = n2_decrypt_ecb,
1194                },
1195        },
1196        {       .name           = "cbc(des3_ede)",
1197                .drv_name       = "cbc-3des",
1198                .block_size     = DES_BLOCK_SIZE,
1199                .enc_type       = (ENC_TYPE_ALG_3DES |
1200                                   ENC_TYPE_CHAINING_CBC),
1201                .ablkcipher     = {
1202                        .ivsize         = DES_BLOCK_SIZE,
1203                        .min_keysize    = 3 * DES_KEY_SIZE,
1204                        .max_keysize    = 3 * DES_KEY_SIZE,
1205                        .setkey         = n2_3des_setkey,
1206                        .encrypt        = n2_encrypt_chaining,
1207                        .decrypt        = n2_decrypt_chaining,
1208                },
1209        },
1210        {       .name           = "cfb(des3_ede)",
1211                .drv_name       = "cfb-3des",
1212                .block_size     = DES_BLOCK_SIZE,
1213                .enc_type       = (ENC_TYPE_ALG_3DES |
1214                                   ENC_TYPE_CHAINING_CFB),
1215                .ablkcipher     = {
1216                        .min_keysize    = 3 * DES_KEY_SIZE,
1217                        .max_keysize    = 3 * DES_KEY_SIZE,
1218                        .setkey         = n2_3des_setkey,
1219                        .encrypt        = n2_encrypt_chaining,
1220                        .decrypt        = n2_decrypt_chaining,
1221                },
1222        },
1223        /* AES: ECB CBC and CTR are supported */
1224        {       .name           = "ecb(aes)",
1225                .drv_name       = "ecb-aes",
1226                .block_size     = AES_BLOCK_SIZE,
1227                .enc_type       = (ENC_TYPE_ALG_AES128 |
1228                                   ENC_TYPE_CHAINING_ECB),
1229                .ablkcipher     = {
1230                        .min_keysize    = AES_MIN_KEY_SIZE,
1231                        .max_keysize    = AES_MAX_KEY_SIZE,
1232                        .setkey         = n2_aes_setkey,
1233                        .encrypt        = n2_encrypt_ecb,
1234                        .decrypt        = n2_decrypt_ecb,
1235                },
1236        },
1237        {       .name           = "cbc(aes)",
1238                .drv_name       = "cbc-aes",
1239                .block_size     = AES_BLOCK_SIZE,
1240                .enc_type       = (ENC_TYPE_ALG_AES128 |
1241                                   ENC_TYPE_CHAINING_CBC),
1242                .ablkcipher     = {
1243                        .ivsize         = AES_BLOCK_SIZE,
1244                        .min_keysize    = AES_MIN_KEY_SIZE,
1245                        .max_keysize    = AES_MAX_KEY_SIZE,
1246                        .setkey         = n2_aes_setkey,
1247                        .encrypt        = n2_encrypt_chaining,
1248                        .decrypt        = n2_decrypt_chaining,
1249                },
1250        },
1251        {       .name           = "ctr(aes)",
1252                .drv_name       = "ctr-aes",
1253                .block_size     = AES_BLOCK_SIZE,
1254                .enc_type       = (ENC_TYPE_ALG_AES128 |
1255                                   ENC_TYPE_CHAINING_COUNTER),
1256                .ablkcipher     = {
1257                        .ivsize         = AES_BLOCK_SIZE,
1258                        .min_keysize    = AES_MIN_KEY_SIZE,
1259                        .max_keysize    = AES_MAX_KEY_SIZE,
1260                        .setkey         = n2_aes_setkey,
1261                        .encrypt        = n2_encrypt_chaining,
1262                        .decrypt        = n2_encrypt_chaining,
1263                },
1264        },
1265
1266};
1267#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1268
1269static LIST_HEAD(cipher_algs);
1270
1271struct n2_hash_tmpl {
1272        const char      *name;
1273        const char      *hash_zero;
1274        const u32       *hash_init;
1275        u8              hw_op_hashsz;
1276        u8              digest_size;
1277        u8              block_size;
1278        u8              auth_type;
1279        u8              hmac_type;
1280};
1281
1282static const char md5_zero[MD5_DIGEST_SIZE] = {
1283        0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
1284        0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
1285};
1286static const u32 md5_init[MD5_HASH_WORDS] = {
1287        cpu_to_le32(0x67452301),
1288        cpu_to_le32(0xefcdab89),
1289        cpu_to_le32(0x98badcfe),
1290        cpu_to_le32(0x10325476),
1291};
1292static const char sha1_zero[SHA1_DIGEST_SIZE] = {
1293        0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
1294        0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
1295        0x07, 0x09
1296};
1297static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
1298        SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
1299};
1300static const char sha256_zero[SHA256_DIGEST_SIZE] = {
1301        0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
1302        0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
1303        0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
1304        0x1b, 0x78, 0x52, 0xb8, 0x55
1305};
1306static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
1307        SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
1308        SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
1309};
1310static const char sha224_zero[SHA224_DIGEST_SIZE] = {
1311        0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
1312        0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
1313        0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
1314        0x2f
1315};
1316static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
1317        SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
1318        SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
1319};
1320
1321static const struct n2_hash_tmpl hash_tmpls[] = {
1322        { .name         = "md5",
1323          .hash_zero    = md5_zero,
1324          .hash_init    = md5_init,
1325          .auth_type    = AUTH_TYPE_MD5,
1326          .hmac_type    = AUTH_TYPE_HMAC_MD5,
1327          .hw_op_hashsz = MD5_DIGEST_SIZE,
1328          .digest_size  = MD5_DIGEST_SIZE,
1329          .block_size   = MD5_HMAC_BLOCK_SIZE },
1330        { .name         = "sha1",
1331          .hash_zero    = sha1_zero,
1332          .hash_init    = sha1_init,
1333          .auth_type    = AUTH_TYPE_SHA1,
1334          .hmac_type    = AUTH_TYPE_HMAC_SHA1,
1335          .hw_op_hashsz = SHA1_DIGEST_SIZE,
1336          .digest_size  = SHA1_DIGEST_SIZE,
1337          .block_size   = SHA1_BLOCK_SIZE },
1338        { .name         = "sha256",
1339          .hash_zero    = sha256_zero,
1340          .hash_init    = sha256_init,
1341          .auth_type    = AUTH_TYPE_SHA256,
1342          .hmac_type    = AUTH_TYPE_HMAC_SHA256,
1343          .hw_op_hashsz = SHA256_DIGEST_SIZE,
1344          .digest_size  = SHA256_DIGEST_SIZE,
1345          .block_size   = SHA256_BLOCK_SIZE },
1346        { .name         = "sha224",
1347          .hash_zero    = sha224_zero,
1348          .hash_init    = sha224_init,
1349          .auth_type    = AUTH_TYPE_SHA256,
1350          .hmac_type    = AUTH_TYPE_RESERVED,
1351          .hw_op_hashsz = SHA256_DIGEST_SIZE,
1352          .digest_size  = SHA224_DIGEST_SIZE,
1353          .block_size   = SHA224_BLOCK_SIZE },
1354};
1355#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1356
1357static LIST_HEAD(ahash_algs);
1358static LIST_HEAD(hmac_algs);
1359
1360static int algs_registered;
1361
1362static void __n2_unregister_algs(void)
1363{
1364        struct n2_cipher_alg *cipher, *cipher_tmp;
1365        struct n2_ahash_alg *alg, *alg_tmp;
1366        struct n2_hmac_alg *hmac, *hmac_tmp;
1367
1368        list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1369                crypto_unregister_alg(&cipher->alg);
1370                list_del(&cipher->entry);
1371                kfree(cipher);
1372        }
1373        list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
1374                crypto_unregister_ahash(&hmac->derived.alg);
1375                list_del(&hmac->derived.entry);
1376                kfree(hmac);
1377        }
1378        list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1379                crypto_unregister_ahash(&alg->alg);
1380                list_del(&alg->entry);
1381                kfree(alg);
1382        }
1383}
1384
1385static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1386{
1387        tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1388        return 0;
1389}
1390
1391static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
1392{
1393        struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1394        struct crypto_alg *alg;
1395        int err;
1396
1397        if (!p)
1398                return -ENOMEM;
1399
1400        alg = &p->alg;
1401
1402        snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1403        snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1404        alg->cra_priority = N2_CRA_PRIORITY;
1405        alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1406                         CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
1407        alg->cra_blocksize = tmpl->block_size;
1408        p->enc_type = tmpl->enc_type;
1409        alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1410        alg->cra_type = &crypto_ablkcipher_type;
1411        alg->cra_u.ablkcipher = tmpl->ablkcipher;
1412        alg->cra_init = n2_cipher_cra_init;
1413        alg->cra_module = THIS_MODULE;
1414
1415        list_add(&p->entry, &cipher_algs);
1416        err = crypto_register_alg(alg);
1417        if (err) {
1418                pr_err("%s alg registration failed\n", alg->cra_name);
1419                list_del(&p->entry);
1420                kfree(p);
1421        } else {
1422                pr_info("%s alg registered\n", alg->cra_name);
1423        }
1424        return err;
1425}
1426
1427static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
1428{
1429        struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1430        struct ahash_alg *ahash;
1431        struct crypto_alg *base;
1432        int err;
1433
1434        if (!p)
1435                return -ENOMEM;
1436
1437        p->child_alg = n2ahash->alg.halg.base.cra_name;
1438        memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
1439        INIT_LIST_HEAD(&p->derived.entry);
1440
1441        ahash = &p->derived.alg;
1442        ahash->digest = n2_hmac_async_digest;
1443        ahash->setkey = n2_hmac_async_setkey;
1444
1445        base = &ahash->halg.base;
1446        snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
1447        snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
1448
1449        base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
1450        base->cra_init = n2_hmac_cra_init;
1451        base->cra_exit = n2_hmac_cra_exit;
1452
1453        list_add(&p->derived.entry, &hmac_algs);
1454        err = crypto_register_ahash(ahash);
1455        if (err) {
1456                pr_err("%s alg registration failed\n", base->cra_name);
1457                list_del(&p->derived.entry);
1458                kfree(p);
1459        } else {
1460                pr_info("%s alg registered\n", base->cra_name);
1461        }
1462        return err;
1463}
1464
1465static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
1466{
1467        struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1468        struct hash_alg_common *halg;
1469        struct crypto_alg *base;
1470        struct ahash_alg *ahash;
1471        int err;
1472
1473        if (!p)
1474                return -ENOMEM;
1475
1476        p->hash_zero = tmpl->hash_zero;
1477        p->hash_init = tmpl->hash_init;
1478        p->auth_type = tmpl->auth_type;
1479        p->hmac_type = tmpl->hmac_type;
1480        p->hw_op_hashsz = tmpl->hw_op_hashsz;
1481        p->digest_size = tmpl->digest_size;
1482
1483        ahash = &p->alg;
1484        ahash->init = n2_hash_async_init;
1485        ahash->update = n2_hash_async_update;
1486        ahash->final = n2_hash_async_final;
1487        ahash->finup = n2_hash_async_finup;
1488        ahash->digest = n2_hash_async_digest;
1489
1490        halg = &ahash->halg;
1491        halg->digestsize = tmpl->digest_size;
1492
1493        base = &halg->base;
1494        snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1495        snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1496        base->cra_priority = N2_CRA_PRIORITY;
1497        base->cra_flags = CRYPTO_ALG_TYPE_AHASH |
1498                          CRYPTO_ALG_KERN_DRIVER_ONLY |
1499                          CRYPTO_ALG_NEED_FALLBACK;
1500        base->cra_blocksize = tmpl->block_size;
1501        base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1502        base->cra_module = THIS_MODULE;
1503        base->cra_init = n2_hash_cra_init;
1504        base->cra_exit = n2_hash_cra_exit;
1505
1506        list_add(&p->entry, &ahash_algs);
1507        err = crypto_register_ahash(ahash);
1508        if (err) {
1509                pr_err("%s alg registration failed\n", base->cra_name);
1510                list_del(&p->entry);
1511                kfree(p);
1512        } else {
1513                pr_info("%s alg registered\n", base->cra_name);
1514        }
1515        if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
1516                err = __n2_register_one_hmac(p);
1517        return err;
1518}
1519
1520static int __devinit n2_register_algs(void)
1521{
1522        int i, err = 0;
1523
1524        mutex_lock(&spu_lock);
1525        if (algs_registered++)
1526                goto out;
1527
1528        for (i = 0; i < NUM_HASH_TMPLS; i++) {
1529                err = __n2_register_one_ahash(&hash_tmpls[i]);
1530                if (err) {
1531                        __n2_unregister_algs();
1532                        goto out;
1533                }
1534        }
1535        for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1536                err = __n2_register_one_cipher(&cipher_tmpls[i]);
1537                if (err) {
1538                        __n2_unregister_algs();
1539                        goto out;
1540                }
1541        }
1542
1543out:
1544        mutex_unlock(&spu_lock);
1545        return err;
1546}
1547
1548static void __devexit n2_unregister_algs(void)
1549{
1550        mutex_lock(&spu_lock);
1551        if (!--algs_registered)
1552                __n2_unregister_algs();
1553        mutex_unlock(&spu_lock);
1554}
1555
1556/* To map CWQ queues to interrupt sources, the hypervisor API provides
1557 * a devino.  This isn't very useful to us because all of the
1558 * interrupts listed in the device_node have been translated to
1559 * Linux virtual IRQ cookie numbers.
1560 *
1561 * So we have to back-translate, going through the 'intr' and 'ino'
1562 * property tables of the n2cp MDESC node, matching it with the OF
1563 * 'interrupts' property entries, in order to to figure out which
1564 * devino goes to which already-translated IRQ.
1565 */
1566static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
1567                             unsigned long dev_ino)
1568{
1569        const unsigned int *dev_intrs;
1570        unsigned int intr;
1571        int i;
1572
1573        for (i = 0; i < ip->num_intrs; i++) {
1574                if (ip->ino_table[i].ino == dev_ino)
1575                        break;
1576        }
1577        if (i == ip->num_intrs)
1578                return -ENODEV;
1579
1580        intr = ip->ino_table[i].intr;
1581
1582        dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
1583        if (!dev_intrs)
1584                return -ENODEV;
1585
1586        for (i = 0; i < dev->archdata.num_irqs; i++) {
1587                if (dev_intrs[i] == intr)
1588                        return i;
1589        }
1590
1591        return -ENODEV;
1592}
1593
1594static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
1595                       const char *irq_name, struct spu_queue *p,
1596                       irq_handler_t handler)
1597{
1598        unsigned long herr;
1599        int index;
1600
1601        herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1602        if (herr)
1603                return -EINVAL;
1604
1605        index = find_devino_index(dev, ip, p->devino);
1606        if (index < 0)
1607                return index;
1608
1609        p->irq = dev->archdata.irqs[index];
1610
1611        sprintf(p->irq_name, "%s-%d", irq_name, index);
1612
1613        return request_irq(p->irq, handler, 0, p->irq_name, p);
1614}
1615
1616static struct kmem_cache *queue_cache[2];
1617
1618static void *new_queue(unsigned long q_type)
1619{
1620        return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1621}
1622
1623static void free_queue(void *p, unsigned long q_type)
1624{
1625        return kmem_cache_free(queue_cache[q_type - 1], p);
1626}
1627
1628static int queue_cache_init(void)
1629{
1630        if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1631                queue_cache[HV_NCS_QTYPE_MAU - 1] =
1632                        kmem_cache_create("mau_queue",
1633                                          (MAU_NUM_ENTRIES *
1634                                           MAU_ENTRY_SIZE),
1635                                          MAU_ENTRY_SIZE, 0, NULL);
1636        if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1637                return -ENOMEM;
1638
1639        if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1640                queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1641                        kmem_cache_create("cwq_queue",
1642                                          (CWQ_NUM_ENTRIES *
1643                                           CWQ_ENTRY_SIZE),
1644                                          CWQ_ENTRY_SIZE, 0, NULL);
1645        if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1646                kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1647                return -ENOMEM;
1648        }
1649        return 0;
1650}
1651
1652static void queue_cache_destroy(void)
1653{
1654        kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1655        kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1656}
1657
1658static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1659{
1660        cpumask_var_t old_allowed;
1661        unsigned long hv_ret;
1662
1663        if (cpumask_empty(&p->sharing))
1664                return -EINVAL;
1665
1666        if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1667                return -ENOMEM;
1668
1669        cpumask_copy(old_allowed, &current->cpus_allowed);
1670
1671        set_cpus_allowed_ptr(current, &p->sharing);
1672
1673        hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1674                                 CWQ_NUM_ENTRIES, &p->qhandle);
1675        if (!hv_ret)
1676                sun4v_ncs_sethead_marker(p->qhandle, 0);
1677
1678        set_cpus_allowed_ptr(current, old_allowed);
1679
1680        free_cpumask_var(old_allowed);
1681
1682        return (hv_ret ? -EINVAL : 0);
1683}
1684
1685static int spu_queue_setup(struct spu_queue *p)
1686{
1687        int err;
1688
1689        p->q = new_queue(p->q_type);
1690        if (!p->q)
1691                return -ENOMEM;
1692
1693        err = spu_queue_register(p, p->q_type);
1694        if (err) {
1695                free_queue(p->q, p->q_type);
1696                p->q = NULL;
1697        }
1698
1699        return err;
1700}
1701
1702static void spu_queue_destroy(struct spu_queue *p)
1703{
1704        unsigned long hv_ret;
1705
1706        if (!p->q)
1707                return;
1708
1709        hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1710
1711        if (!hv_ret)
1712                free_queue(p->q, p->q_type);
1713}
1714
1715static void spu_list_destroy(struct list_head *list)
1716{
1717        struct spu_queue *p, *n;
1718
1719        list_for_each_entry_safe(p, n, list, list) {
1720                int i;
1721
1722                for (i = 0; i < NR_CPUS; i++) {
1723                        if (cpu_to_cwq[i] == p)
1724                                cpu_to_cwq[i] = NULL;
1725                }
1726
1727                if (p->irq) {
1728                        free_irq(p->irq, p);
1729                        p->irq = 0;
1730                }
1731                spu_queue_destroy(p);
1732                list_del(&p->list);
1733                kfree(p);
1734        }
1735}
1736
1737/* Walk the backward arcs of a CWQ 'exec-unit' node,
1738 * gathering cpu membership information.
1739 */
1740static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1741                               struct platform_device *dev,
1742                               u64 node, struct spu_queue *p,
1743                               struct spu_queue **table)
1744{
1745        u64 arc;
1746
1747        mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1748                u64 tgt = mdesc_arc_target(mdesc, arc);
1749                const char *name = mdesc_node_name(mdesc, tgt);
1750                const u64 *id;
1751
1752                if (strcmp(name, "cpu"))
1753                        continue;
1754                id = mdesc_get_property(mdesc, tgt, "id", NULL);
1755                if (table[*id] != NULL) {
1756                        dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
1757                                dev->dev.of_node->full_name);
1758                        return -EINVAL;
1759                }
1760                cpu_set(*id, p->sharing);
1761                table[*id] = p;
1762        }
1763        return 0;
1764}
1765
1766/* Process an 'exec-unit' MDESC node of type 'cwq'.  */
1767static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1768                            struct platform_device *dev, struct mdesc_handle *mdesc,
1769                            u64 node, const char *iname, unsigned long q_type,
1770                            irq_handler_t handler, struct spu_queue **table)
1771{
1772        struct spu_queue *p;
1773        int err;
1774
1775        p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1776        if (!p) {
1777                dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
1778                        dev->dev.of_node->full_name);
1779                return -ENOMEM;
1780        }
1781
1782        cpus_clear(p->sharing);
1783        spin_lock_init(&p->lock);
1784        p->q_type = q_type;
1785        INIT_LIST_HEAD(&p->jobs);
1786        list_add(&p->list, list);
1787
1788        err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1789        if (err)
1790                return err;
1791
1792        err = spu_queue_setup(p);
1793        if (err)
1794                return err;
1795
1796        return spu_map_ino(dev, ip, iname, p, handler);
1797}
1798
1799static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
1800                          struct spu_mdesc_info *ip, struct list_head *list,
1801                          const char *exec_name, unsigned long q_type,
1802                          irq_handler_t handler, struct spu_queue **table)
1803{
1804        int err = 0;
1805        u64 node;
1806
1807        mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1808                const char *type;
1809
1810                type = mdesc_get_property(mdesc, node, "type", NULL);
1811                if (!type || strcmp(type, exec_name))
1812                        continue;
1813
1814                err = handle_exec_unit(ip, list, dev, mdesc, node,
1815                                       exec_name, q_type, handler, table);
1816                if (err) {
1817                        spu_list_destroy(list);
1818                        break;
1819                }
1820        }
1821
1822        return err;
1823}
1824
1825static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,
1826                                   struct spu_mdesc_info *ip)
1827{
1828        const u64 *ino;
1829        int ino_len;
1830        int i;
1831
1832        ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1833        if (!ino) {
1834                printk("NO 'ino'\n");
1835                return -ENODEV;
1836        }
1837
1838        ip->num_intrs = ino_len / sizeof(u64);
1839        ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1840                                 ip->num_intrs),
1841                                GFP_KERNEL);
1842        if (!ip->ino_table)
1843                return -ENOMEM;
1844
1845        for (i = 0; i < ip->num_intrs; i++) {
1846                struct ino_blob *b = &ip->ino_table[i];
1847                b->intr = i + 1;
1848                b->ino = ino[i];
1849        }
1850
1851        return 0;
1852}
1853
1854static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1855                                          struct platform_device *dev,
1856                                          struct spu_mdesc_info *ip,
1857                                          const char *node_name)
1858{
1859        const unsigned int *reg;
1860        u64 node;
1861
1862        reg = of_get_property(dev->dev.of_node, "reg", NULL);
1863        if (!reg)
1864                return -ENODEV;
1865
1866        mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1867                const char *name;
1868                const u64 *chdl;
1869
1870                name = mdesc_get_property(mdesc, node, "name", NULL);
1871                if (!name || strcmp(name, node_name))
1872                        continue;
1873                chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1874                if (!chdl || (*chdl != *reg))
1875                        continue;
1876                ip->cfg_handle = *chdl;
1877                return get_irq_props(mdesc, node, ip);
1878        }
1879
1880        return -ENODEV;
1881}
1882
1883static unsigned long n2_spu_hvapi_major;
1884static unsigned long n2_spu_hvapi_minor;
1885
1886static int __devinit n2_spu_hvapi_register(void)
1887{
1888        int err;
1889
1890        n2_spu_hvapi_major = 2;
1891        n2_spu_hvapi_minor = 0;
1892
1893        err = sun4v_hvapi_register(HV_GRP_NCS,
1894                                   n2_spu_hvapi_major,
1895                                   &n2_spu_hvapi_minor);
1896
1897        if (!err)
1898                pr_info("Registered NCS HVAPI version %lu.%lu\n",
1899                        n2_spu_hvapi_major,
1900                        n2_spu_hvapi_minor);
1901
1902        return err;
1903}
1904
1905static void n2_spu_hvapi_unregister(void)
1906{
1907        sun4v_hvapi_unregister(HV_GRP_NCS);
1908}
1909
1910static int global_ref;
1911
1912static int __devinit grab_global_resources(void)
1913{
1914        int err = 0;
1915
1916        mutex_lock(&spu_lock);
1917
1918        if (global_ref++)
1919                goto out;
1920
1921        err = n2_spu_hvapi_register();
1922        if (err)
1923                goto out;
1924
1925        err = queue_cache_init();
1926        if (err)
1927                goto out_hvapi_release;
1928
1929        err = -ENOMEM;
1930        cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1931                             GFP_KERNEL);
1932        if (!cpu_to_cwq)
1933                goto out_queue_cache_destroy;
1934
1935        cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1936                             GFP_KERNEL);
1937        if (!cpu_to_mau)
1938                goto out_free_cwq_table;
1939
1940        err = 0;
1941
1942out:
1943        if (err)
1944                global_ref--;
1945        mutex_unlock(&spu_lock);
1946        return err;
1947
1948out_free_cwq_table:
1949        kfree(cpu_to_cwq);
1950        cpu_to_cwq = NULL;
1951
1952out_queue_cache_destroy:
1953        queue_cache_destroy();
1954
1955out_hvapi_release:
1956        n2_spu_hvapi_unregister();
1957        goto out;
1958}
1959
1960static void release_global_resources(void)
1961{
1962        mutex_lock(&spu_lock);
1963        if (!--global_ref) {
1964                kfree(cpu_to_cwq);
1965                cpu_to_cwq = NULL;
1966
1967                kfree(cpu_to_mau);
1968                cpu_to_mau = NULL;
1969
1970                queue_cache_destroy();
1971                n2_spu_hvapi_unregister();
1972        }
1973        mutex_unlock(&spu_lock);
1974}
1975
1976static struct n2_crypto * __devinit alloc_n2cp(void)
1977{
1978        struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1979
1980        if (np)
1981                INIT_LIST_HEAD(&np->cwq_list);
1982
1983        return np;
1984}
1985
1986static void free_n2cp(struct n2_crypto *np)
1987{
1988        if (np->cwq_info.ino_table) {
1989                kfree(np->cwq_info.ino_table);
1990                np->cwq_info.ino_table = NULL;
1991        }
1992
1993        kfree(np);
1994}
1995
1996static void __devinit n2_spu_driver_version(void)
1997{
1998        static int n2_spu_version_printed;
1999
2000        if (n2_spu_version_printed++ == 0)
2001                pr_info("%s", version);
2002}
2003
2004static int __devinit n2_crypto_probe(struct platform_device *dev)
2005{
2006        struct mdesc_handle *mdesc;
2007        const char *full_name;
2008        struct n2_crypto *np;
2009        int err;
2010
2011        n2_spu_driver_version();
2012
2013        full_name = dev->dev.of_node->full_name;
2014        pr_info("Found N2CP at %s\n", full_name);
2015
2016        np = alloc_n2cp();
2017        if (!np) {
2018                dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
2019                        full_name);
2020                return -ENOMEM;
2021        }
2022
2023        err = grab_global_resources();
2024        if (err) {
2025                dev_err(&dev->dev, "%s: Unable to grab "
2026                        "global resources.\n", full_name);
2027                goto out_free_n2cp;
2028        }
2029
2030        mdesc = mdesc_grab();
2031
2032        if (!mdesc) {
2033                dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2034                        full_name);
2035                err = -ENODEV;
2036                goto out_free_global;
2037        }
2038        err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
2039        if (err) {
2040                dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2041                        full_name);
2042                mdesc_release(mdesc);
2043                goto out_free_global;
2044        }
2045
2046        err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
2047                             "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
2048                             cpu_to_cwq);
2049        mdesc_release(mdesc);
2050
2051        if (err) {
2052                dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
2053                        full_name);
2054                goto out_free_global;
2055        }
2056
2057        err = n2_register_algs();
2058        if (err) {
2059                dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
2060                        full_name);
2061                goto out_free_spu_list;
2062        }
2063
2064        dev_set_drvdata(&dev->dev, np);
2065
2066        return 0;
2067
2068out_free_spu_list:
2069        spu_list_destroy(&np->cwq_list);
2070
2071out_free_global:
2072        release_global_resources();
2073
2074out_free_n2cp:
2075        free_n2cp(np);
2076
2077        return err;
2078}
2079
2080static int __devexit n2_crypto_remove(struct platform_device *dev)
2081{
2082        struct n2_crypto *np = dev_get_drvdata(&dev->dev);
2083
2084        n2_unregister_algs();
2085
2086        spu_list_destroy(&np->cwq_list);
2087
2088        release_global_resources();
2089
2090        free_n2cp(np);
2091
2092        return 0;
2093}
2094
2095static struct n2_mau * __devinit alloc_ncp(void)
2096{
2097        struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
2098
2099        if (mp)
2100                INIT_LIST_HEAD(&mp->mau_list);
2101
2102        return mp;
2103}
2104
2105static void free_ncp(struct n2_mau *mp)
2106{
2107        if (mp->mau_info.ino_table) {
2108                kfree(mp->mau_info.ino_table);
2109                mp->mau_info.ino_table = NULL;
2110        }
2111
2112        kfree(mp);
2113}
2114
2115static int __devinit n2_mau_probe(struct platform_device *dev)
2116{
2117        struct mdesc_handle *mdesc;
2118        const char *full_name;
2119        struct n2_mau *mp;
2120        int err;
2121
2122        n2_spu_driver_version();
2123
2124        full_name = dev->dev.of_node->full_name;
2125        pr_info("Found NCP at %s\n", full_name);
2126
2127        mp = alloc_ncp();
2128        if (!mp) {
2129                dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
2130                        full_name);
2131                return -ENOMEM;
2132        }
2133
2134        err = grab_global_resources();
2135        if (err) {
2136                dev_err(&dev->dev, "%s: Unable to grab "
2137                        "global resources.\n", full_name);
2138                goto out_free_ncp;
2139        }
2140
2141        mdesc = mdesc_grab();
2142
2143        if (!mdesc) {
2144                dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2145                        full_name);
2146                err = -ENODEV;
2147                goto out_free_global;
2148        }
2149
2150        err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
2151        if (err) {
2152                dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2153                        full_name);
2154                mdesc_release(mdesc);
2155                goto out_free_global;
2156        }
2157
2158        err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
2159                             "mau", HV_NCS_QTYPE_MAU, mau_intr,
2160                             cpu_to_mau);
2161        mdesc_release(mdesc);
2162
2163        if (err) {
2164                dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
2165                        full_name);
2166                goto out_free_global;
2167        }
2168
2169        dev_set_drvdata(&dev->dev, mp);
2170
2171        return 0;
2172
2173out_free_global:
2174        release_global_resources();
2175
2176out_free_ncp:
2177        free_ncp(mp);
2178
2179        return err;
2180}
2181
2182static int __devexit n2_mau_remove(struct platform_device *dev)
2183{
2184        struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2185
2186        spu_list_destroy(&mp->mau_list);
2187
2188        release_global_resources();
2189
2190        free_ncp(mp);
2191
2192        return 0;
2193}
2194
2195static struct of_device_id n2_crypto_match[] = {
2196        {
2197                .name = "n2cp",
2198                .compatible = "SUNW,n2-cwq",
2199        },
2200        {
2201                .name = "n2cp",
2202                .compatible = "SUNW,vf-cwq",
2203        },
2204        {
2205                .name = "n2cp",
2206                .compatible = "SUNW,kt-cwq",
2207        },
2208        {},
2209};
2210
2211MODULE_DEVICE_TABLE(of, n2_crypto_match);
2212
2213static struct platform_driver n2_crypto_driver = {
2214        .driver = {
2215                .name           =       "n2cp",
2216                .owner          =       THIS_MODULE,
2217                .of_match_table =       n2_crypto_match,
2218        },
2219        .probe          =       n2_crypto_probe,
2220        .remove         =       __devexit_p(n2_crypto_remove),
2221};
2222
2223static struct of_device_id n2_mau_match[] = {
2224        {
2225                .name = "ncp",
2226                .compatible = "SUNW,n2-mau",
2227        },
2228        {
2229                .name = "ncp",
2230                .compatible = "SUNW,vf-mau",
2231        },
2232        {
2233                .name = "ncp",
2234                .compatible = "SUNW,kt-mau",
2235        },
2236        {},
2237};
2238
2239MODULE_DEVICE_TABLE(of, n2_mau_match);
2240
2241static struct platform_driver n2_mau_driver = {
2242        .driver = {
2243                .name           =       "ncp",
2244                .owner          =       THIS_MODULE,
2245                .of_match_table =       n2_mau_match,
2246        },
2247        .probe          =       n2_mau_probe,
2248        .remove         =       __devexit_p(n2_mau_remove),
2249};
2250
2251static int __init n2_init(void)
2252{
2253        int err = platform_driver_register(&n2_crypto_driver);
2254
2255        if (!err) {
2256                err = platform_driver_register(&n2_mau_driver);
2257                if (err)
2258                        platform_driver_unregister(&n2_crypto_driver);
2259        }
2260        return err;
2261}
2262
2263static void __exit n2_exit(void)
2264{
2265        platform_driver_unregister(&n2_mau_driver);
2266        platform_driver_unregister(&n2_crypto_driver);
2267}
2268
2269module_init(n2_init);
2270module_exit(n2_exit);
2271
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.