linux/drivers/crypto/nx/nx-842-pseries.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Driver for IBM Power 842 compression accelerator
   4 *
   5 * Copyright (C) IBM Corporation, 2012
   6 *
   7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
   8 *          Seth Jennings <sjenning@linux.vnet.ibm.com>
   9 */
  10
  11#include <asm/vio.h>
  12
  13#include "nx-842.h"
  14#include "nx_csbcpb.h" /* struct nx_csbcpb */
  15
  16MODULE_LICENSE("GPL");
  17MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
  18MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
  19MODULE_ALIAS_CRYPTO("842");
  20MODULE_ALIAS_CRYPTO("842-nx");
  21
  22static struct nx842_constraints nx842_pseries_constraints = {
  23        .alignment =    DDE_BUFFER_ALIGN,
  24        .multiple =     DDE_BUFFER_LAST_MULT,
  25        .minimum =      DDE_BUFFER_LAST_MULT,
  26        .maximum =      PAGE_SIZE, /* dynamic, max_sync_size */
  27};
  28
  29static int check_constraints(unsigned long buf, unsigned int *len, bool in)
  30{
  31        if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
  32                pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
  33                         in ? "input" : "output", buf,
  34                         nx842_pseries_constraints.alignment);
  35                return -EINVAL;
  36        }
  37        if (*len % nx842_pseries_constraints.multiple) {
  38                pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
  39                         in ? "input" : "output", *len,
  40                         nx842_pseries_constraints.multiple);
  41                if (in)
  42                        return -EINVAL;
  43                *len = round_down(*len, nx842_pseries_constraints.multiple);
  44        }
  45        if (*len < nx842_pseries_constraints.minimum) {
  46                pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
  47                         in ? "input" : "output", *len,
  48                         nx842_pseries_constraints.minimum);
  49                return -EINVAL;
  50        }
  51        if (*len > nx842_pseries_constraints.maximum) {
  52                pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
  53                         in ? "input" : "output", *len,
  54                         nx842_pseries_constraints.maximum);
  55                if (in)
  56                        return -EINVAL;
  57                *len = nx842_pseries_constraints.maximum;
  58        }
  59        return 0;
  60}
  61
  62/* I assume we need to align the CSB? */
  63#define WORKMEM_ALIGN   (256)
  64
  65struct nx842_workmem {
  66        /* scatterlist */
  67        char slin[4096];
  68        char slout[4096];
  69        /* coprocessor status/parameter block */
  70        struct nx_csbcpb csbcpb;
  71
  72        char padding[WORKMEM_ALIGN];
  73} __aligned(WORKMEM_ALIGN);
  74
  75/* Macros for fields within nx_csbcpb */
  76/* Check the valid bit within the csbcpb valid field */
  77#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
  78
  79/* CE macros operate on the completion_extension field bits in the csbcpb.
  80 * CE0 0=full completion, 1=partial completion
  81 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
  82 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
  83#define NX842_CSBCPB_CE0(x)     (x & BIT_MASK(7))
  84#define NX842_CSBCPB_CE1(x)     (x & BIT_MASK(6))
  85#define NX842_CSBCPB_CE2(x)     (x & BIT_MASK(5))
  86
  87/* The NX unit accepts data only on 4K page boundaries */
  88#define NX842_HW_PAGE_SIZE      (4096)
  89#define NX842_HW_PAGE_MASK      (~(NX842_HW_PAGE_SIZE-1))
  90
  91struct ibm_nx842_counters {
  92        atomic64_t comp_complete;
  93        atomic64_t comp_failed;
  94        atomic64_t decomp_complete;
  95        atomic64_t decomp_failed;
  96        atomic64_t swdecomp;
  97        atomic64_t comp_times[32];
  98        atomic64_t decomp_times[32];
  99};
 100
 101static struct nx842_devdata {
 102        struct vio_dev *vdev;
 103        struct device *dev;
 104        struct ibm_nx842_counters *counters;
 105        unsigned int max_sg_len;
 106        unsigned int max_sync_size;
 107        unsigned int max_sync_sg;
 108} __rcu *devdata;
 109static DEFINE_SPINLOCK(devdata_mutex);
 110
 111#define NX842_COUNTER_INC(_x) \
 112static inline void nx842_inc_##_x( \
 113        const struct nx842_devdata *dev) { \
 114        if (dev) \
 115                atomic64_inc(&dev->counters->_x); \
 116}
 117NX842_COUNTER_INC(comp_complete);
 118NX842_COUNTER_INC(comp_failed);
 119NX842_COUNTER_INC(decomp_complete);
 120NX842_COUNTER_INC(decomp_failed);
 121NX842_COUNTER_INC(swdecomp);
 122
 123#define NX842_HIST_SLOTS 16
 124
 125static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
 126{
 127        int bucket = fls(time);
 128
 129        if (bucket)
 130                bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
 131
 132        atomic64_inc(&times[bucket]);
 133}
 134
 135/* NX unit operation flags */
 136#define NX842_OP_COMPRESS       0x0
 137#define NX842_OP_CRC            0x1
 138#define NX842_OP_DECOMPRESS     0x2
 139#define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
 140#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
 141#define NX842_OP_ASYNC          (1<<23)
 142#define NX842_OP_NOTIFY         (1<<22)
 143#define NX842_OP_NOTIFY_INT(x)  ((x & 0xff)<<8)
 144
 145static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
 146{
 147        /* No use of DMA mappings within the driver. */
 148        return 0;
 149}
 150
 151struct nx842_slentry {
 152        __be64 ptr; /* Real address (use __pa()) */
 153        __be64 len;
 154};
 155
 156/* pHyp scatterlist entry */
 157struct nx842_scatterlist {
 158        int entry_nr; /* number of slentries */
 159        struct nx842_slentry *entries; /* ptr to array of slentries */
 160};
 161
 162/* Does not include sizeof(entry_nr) in the size */
 163static inline unsigned long nx842_get_scatterlist_size(
 164                                struct nx842_scatterlist *sl)
 165{
 166        return sl->entry_nr * sizeof(struct nx842_slentry);
 167}
 168
 169static int nx842_build_scatterlist(unsigned long buf, int len,
 170                        struct nx842_scatterlist *sl)
 171{
 172        unsigned long entrylen;
 173        struct nx842_slentry *entry;
 174
 175        sl->entry_nr = 0;
 176
 177        entry = sl->entries;
 178        while (len) {
 179                entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
 180                entrylen = min_t(int, len,
 181                                 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
 182                entry->len = cpu_to_be64(entrylen);
 183
 184                len -= entrylen;
 185                buf += entrylen;
 186
 187                sl->entry_nr++;
 188                entry++;
 189        }
 190
 191        return 0;
 192}
 193
 194static int nx842_validate_result(struct device *dev,
 195        struct cop_status_block *csb)
 196{
 197        /* The csb must be valid after returning from vio_h_cop_sync */
 198        if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
 199                dev_err(dev, "%s: cspcbp not valid upon completion.\n",
 200                                __func__);
 201                dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
 202                                csb->valid,
 203                                csb->crb_seq_number,
 204                                csb->completion_code,
 205                                csb->completion_extension);
 206                dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
 207                                be32_to_cpu(csb->processed_byte_count),
 208                                (unsigned long)be64_to_cpu(csb->address));
 209                return -EIO;
 210        }
 211
 212        /* Check return values from the hardware in the CSB */
 213        switch (csb->completion_code) {
 214        case 0: /* Completed without error */
 215                break;
 216        case 64: /* Compression ok, but output larger than input */
 217                dev_dbg(dev, "%s: output size larger than input size\n",
 218                                        __func__);
 219                break;
 220        case 13: /* Output buffer too small */
 221                dev_dbg(dev, "%s: Out of space in output buffer\n",
 222                                        __func__);
 223                return -ENOSPC;
 224        case 65: /* Calculated CRC doesn't match the passed value */
 225                dev_dbg(dev, "%s: CRC mismatch for decompression\n",
 226                                        __func__);
 227                return -EINVAL;
 228        case 66: /* Input data contains an illegal template field */
 229        case 67: /* Template indicates data past the end of the input stream */
 230                dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
 231                                        __func__, csb->completion_code);
 232                return -EINVAL;
 233        default:
 234                dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
 235                                        __func__, csb->completion_code);
 236                return -EIO;
 237        }
 238
 239        /* Hardware sanity check */
 240        if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
 241                dev_err(dev, "%s: No error returned by hardware, but "
 242                                "data returned is unusable, contact support.\n"
 243                                "(Additional info: csbcbp->processed bytes "
 244                                "does not specify processed bytes for the "
 245                                "target buffer.)\n", __func__);
 246                return -EIO;
 247        }
 248
 249        return 0;
 250}
 251
 252/**
 253 * nx842_pseries_compress - Compress data using the 842 algorithm
 254 *
 255 * Compression provide by the NX842 coprocessor on IBM Power systems.
 256 * The input buffer is compressed and the result is stored in the
 257 * provided output buffer.
 258 *
 259 * Upon return from this function @outlen contains the length of the
 260 * compressed data.  If there is an error then @outlen will be 0 and an
 261 * error will be specified by the return code from this function.
 262 *
 263 * @in: Pointer to input buffer
 264 * @inlen: Length of input buffer
 265 * @out: Pointer to output buffer
 266 * @outlen: Length of output buffer
 267 * @wrkmem: ptr to buffer for working memory, size determined by
 268 *          nx842_pseries_driver.workmem_size
 269 *
 270 * Returns:
 271 *   0          Success, output of length @outlen stored in the buffer at @out
 272 *   -ENOMEM    Unable to allocate internal buffers
 273 *   -ENOSPC    Output buffer is to small
 274 *   -EIO       Internal error
 275 *   -ENODEV    Hardware unavailable
 276 */
 277static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
 278                                  unsigned char *out, unsigned int *outlen,
 279                                  void *wmem)
 280{
 281        struct nx842_devdata *local_devdata;
 282        struct device *dev = NULL;
 283        struct nx842_workmem *workmem;
 284        struct nx842_scatterlist slin, slout;
 285        struct nx_csbcpb *csbcpb;
 286        int ret = 0;
 287        unsigned long inbuf, outbuf;
 288        struct vio_pfo_op op = {
 289                .done = NULL,
 290                .handle = 0,
 291                .timeout = 0,
 292        };
 293        unsigned long start = get_tb();
 294
 295        inbuf = (unsigned long)in;
 296        if (check_constraints(inbuf, &inlen, true))
 297                return -EINVAL;
 298
 299        outbuf = (unsigned long)out;
 300        if (check_constraints(outbuf, outlen, false))
 301                return -EINVAL;
 302
 303        rcu_read_lock();
 304        local_devdata = rcu_dereference(devdata);
 305        if (!local_devdata || !local_devdata->dev) {
 306                rcu_read_unlock();
 307                return -ENODEV;
 308        }
 309        dev = local_devdata->dev;
 310
 311        /* Init scatterlist */
 312        workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 313        slin.entries = (struct nx842_slentry *)workmem->slin;
 314        slout.entries = (struct nx842_slentry *)workmem->slout;
 315
 316        /* Init operation */
 317        op.flags = NX842_OP_COMPRESS_CRC;
 318        csbcpb = &workmem->csbcpb;
 319        memset(csbcpb, 0, sizeof(*csbcpb));
 320        op.csbcpb = nx842_get_pa(csbcpb);
 321
 322        if ((inbuf & NX842_HW_PAGE_MASK) ==
 323            ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 324                /* Create direct DDE */
 325                op.in = nx842_get_pa((void *)inbuf);
 326                op.inlen = inlen;
 327        } else {
 328                /* Create indirect DDE (scatterlist) */
 329                nx842_build_scatterlist(inbuf, inlen, &slin);
 330                op.in = nx842_get_pa(slin.entries);
 331                op.inlen = -nx842_get_scatterlist_size(&slin);
 332        }
 333
 334        if ((outbuf & NX842_HW_PAGE_MASK) ==
 335            ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 336                /* Create direct DDE */
 337                op.out = nx842_get_pa((void *)outbuf);
 338                op.outlen = *outlen;
 339        } else {
 340                /* Create indirect DDE (scatterlist) */
 341                nx842_build_scatterlist(outbuf, *outlen, &slout);
 342                op.out = nx842_get_pa(slout.entries);
 343                op.outlen = -nx842_get_scatterlist_size(&slout);
 344        }
 345
 346        dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 347                __func__, (unsigned long)op.in, (long)op.inlen,
 348                (unsigned long)op.out, (long)op.outlen);
 349
 350        /* Send request to pHyp */
 351        ret = vio_h_cop_sync(local_devdata->vdev, &op);
 352
 353        /* Check for pHyp error */
 354        if (ret) {
 355                dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 356                        __func__, ret, op.hcall_err);
 357                ret = -EIO;
 358                goto unlock;
 359        }
 360
 361        /* Check for hardware error */
 362        ret = nx842_validate_result(dev, &csbcpb->csb);
 363        if (ret)
 364                goto unlock;
 365
 366        *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 367        dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
 368
 369unlock:
 370        if (ret)
 371                nx842_inc_comp_failed(local_devdata);
 372        else {
 373                nx842_inc_comp_complete(local_devdata);
 374                ibm_nx842_incr_hist(local_devdata->counters->comp_times,
 375                        (get_tb() - start) / tb_ticks_per_usec);
 376        }
 377        rcu_read_unlock();
 378        return ret;
 379}
 380
 381/**
 382 * nx842_pseries_decompress - Decompress data using the 842 algorithm
 383 *
 384 * Decompression provide by the NX842 coprocessor on IBM Power systems.
 385 * The input buffer is decompressed and the result is stored in the
 386 * provided output buffer.  The size allocated to the output buffer is
 387 * provided by the caller of this function in @outlen.  Upon return from
 388 * this function @outlen contains the length of the decompressed data.
 389 * If there is an error then @outlen will be 0 and an error will be
 390 * specified by the return code from this function.
 391 *
 392 * @in: Pointer to input buffer
 393 * @inlen: Length of input buffer
 394 * @out: Pointer to output buffer
 395 * @outlen: Length of output buffer
 396 * @wrkmem: ptr to buffer for working memory, size determined by
 397 *          nx842_pseries_driver.workmem_size
 398 *
 399 * Returns:
 400 *   0          Success, output of length @outlen stored in the buffer at @out
 401 *   -ENODEV    Hardware decompression device is unavailable
 402 *   -ENOMEM    Unable to allocate internal buffers
 403 *   -ENOSPC    Output buffer is to small
 404 *   -EINVAL    Bad input data encountered when attempting decompress
 405 *   -EIO       Internal error
 406 */
 407static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
 408                                    unsigned char *out, unsigned int *outlen,
 409                                    void *wmem)
 410{
 411        struct nx842_devdata *local_devdata;
 412        struct device *dev = NULL;
 413        struct nx842_workmem *workmem;
 414        struct nx842_scatterlist slin, slout;
 415        struct nx_csbcpb *csbcpb;
 416        int ret = 0;
 417        unsigned long inbuf, outbuf;
 418        struct vio_pfo_op op = {
 419                .done = NULL,
 420                .handle = 0,
 421                .timeout = 0,
 422        };
 423        unsigned long start = get_tb();
 424
 425        /* Ensure page alignment and size */
 426        inbuf = (unsigned long)in;
 427        if (check_constraints(inbuf, &inlen, true))
 428                return -EINVAL;
 429
 430        outbuf = (unsigned long)out;
 431        if (check_constraints(outbuf, outlen, false))
 432                return -EINVAL;
 433
 434        rcu_read_lock();
 435        local_devdata = rcu_dereference(devdata);
 436        if (!local_devdata || !local_devdata->dev) {
 437                rcu_read_unlock();
 438                return -ENODEV;
 439        }
 440        dev = local_devdata->dev;
 441
 442        workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 443
 444        /* Init scatterlist */
 445        slin.entries = (struct nx842_slentry *)workmem->slin;
 446        slout.entries = (struct nx842_slentry *)workmem->slout;
 447
 448        /* Init operation */
 449        op.flags = NX842_OP_DECOMPRESS_CRC;
 450        csbcpb = &workmem->csbcpb;
 451        memset(csbcpb, 0, sizeof(*csbcpb));
 452        op.csbcpb = nx842_get_pa(csbcpb);
 453
 454        if ((inbuf & NX842_HW_PAGE_MASK) ==
 455            ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 456                /* Create direct DDE */
 457                op.in = nx842_get_pa((void *)inbuf);
 458                op.inlen = inlen;
 459        } else {
 460                /* Create indirect DDE (scatterlist) */
 461                nx842_build_scatterlist(inbuf, inlen, &slin);
 462                op.in = nx842_get_pa(slin.entries);
 463                op.inlen = -nx842_get_scatterlist_size(&slin);
 464        }
 465
 466        if ((outbuf & NX842_HW_PAGE_MASK) ==
 467            ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 468                /* Create direct DDE */
 469                op.out = nx842_get_pa((void *)outbuf);
 470                op.outlen = *outlen;
 471        } else {
 472                /* Create indirect DDE (scatterlist) */
 473                nx842_build_scatterlist(outbuf, *outlen, &slout);
 474                op.out = nx842_get_pa(slout.entries);
 475                op.outlen = -nx842_get_scatterlist_size(&slout);
 476        }
 477
 478        dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 479                __func__, (unsigned long)op.in, (long)op.inlen,
 480                (unsigned long)op.out, (long)op.outlen);
 481
 482        /* Send request to pHyp */
 483        ret = vio_h_cop_sync(local_devdata->vdev, &op);
 484
 485        /* Check for pHyp error */
 486        if (ret) {
 487                dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 488                        __func__, ret, op.hcall_err);
 489                goto unlock;
 490        }
 491
 492        /* Check for hardware error */
 493        ret = nx842_validate_result(dev, &csbcpb->csb);
 494        if (ret)
 495                goto unlock;
 496
 497        *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 498
 499unlock:
 500        if (ret)
 501                /* decompress fail */
 502                nx842_inc_decomp_failed(local_devdata);
 503        else {
 504                nx842_inc_decomp_complete(local_devdata);
 505                ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
 506                        (get_tb() - start) / tb_ticks_per_usec);
 507        }
 508
 509        rcu_read_unlock();
 510        return ret;
 511}
 512
 513/**
 514 * nx842_OF_set_defaults -- Set default (disabled) values for devdata
 515 *
 516 * @devdata - struct nx842_devdata to update
 517 *
 518 * Returns:
 519 *  0 on success
 520 *  -ENOENT if @devdata ptr is NULL
 521 */
 522static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
 523{
 524        if (devdata) {
 525                devdata->max_sync_size = 0;
 526                devdata->max_sync_sg = 0;
 527                devdata->max_sg_len = 0;
 528                return 0;
 529        } else
 530                return -ENOENT;
 531}
 532
 533/**
 534 * nx842_OF_upd_status -- Check the device info from OF status prop
 535 *
 536 * The status property indicates if the accelerator is enabled.  If the
 537 * device is in the OF tree it indicates that the hardware is present.
 538 * The status field indicates if the device is enabled when the status
 539 * is 'okay'.  Otherwise the device driver will be disabled.
 540 *
 541 * @devdata: struct nx842_devdata to use for dev_info
 542 * @prop: struct property point containing the maxsyncop for the update
 543 *
 544 * Returns:
 545 *  0 - Device is available
 546 *  -ENODEV - Device is not available
 547 */
 548static int nx842_OF_upd_status(struct nx842_devdata *devdata,
 549                               struct property *prop)
 550{
 551        const char *status = (const char *)prop->value;
 552
 553        if (!strncmp(status, "okay", (size_t)prop->length))
 554                return 0;
 555        if (!strncmp(status, "disabled", (size_t)prop->length))
 556                return -ENODEV;
 557        dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
 558
 559        return -EINVAL;
 560}
 561
 562/**
 563 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
 564 *
 565 * Definition of the 'ibm,max-sg-len' OF property:
 566 *  This field indicates the maximum byte length of a scatter list
 567 *  for the platform facility. It is a single cell encoded as with encode-int.
 568 *
 569 * Example:
 570 *  # od -x ibm,max-sg-len
 571 *  0000000 0000 0ff0
 572 *
 573 *  In this example, the maximum byte length of a scatter list is
 574 *  0x0ff0 (4,080).
 575 *
 576 * @devdata - struct nx842_devdata to update
 577 * @prop - struct property point containing the maxsyncop for the update
 578 *
 579 * Returns:
 580 *  0 on success
 581 *  -EINVAL on failure
 582 */
 583static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
 584                                        struct property *prop) {
 585        int ret = 0;
 586        const unsigned int maxsglen = of_read_number(prop->value, 1);
 587
 588        if (prop->length != sizeof(maxsglen)) {
 589                dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
 590                dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
 591                                prop->length, sizeof(maxsglen));
 592                ret = -EINVAL;
 593        } else {
 594                devdata->max_sg_len = min_t(unsigned int,
 595                                            maxsglen, NX842_HW_PAGE_SIZE);
 596        }
 597
 598        return ret;
 599}
 600
 601/**
 602 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
 603 *
 604 * Definition of the 'ibm,max-sync-cop' OF property:
 605 *  Two series of cells.  The first series of cells represents the maximums
 606 *  that can be synchronously compressed. The second series of cells
 607 *  represents the maximums that can be synchronously decompressed.
 608 *  1. The first cell in each series contains the count of the number of
 609 *     data length, scatter list elements pairs that follow \xE2\x80\x93 each being
 610 *     of the form
 611 *    a. One cell data byte length
 612 *    b. One cell total number of scatter list elements
 613 *
 614 * Example:
 615 *  # od -x ibm,max-sync-cop
 616 *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
 617 *  0000020 0000 1000 0000 01fe
 618 *
 619 *  In this example, compression supports 0x1000 (4,096) data byte length
 620 *  and 0x1fe (510) total scatter list elements.  Decompression supports
 621 *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
 622 *  elements.
 623 *
 624 * @devdata - struct nx842_devdata to update
 625 * @prop - struct property point containing the maxsyncop for the update
 626 *
 627 * Returns:
 628 *  0 on success
 629 *  -EINVAL on failure
 630 */
 631static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
 632                                        struct property *prop) {
 633        int ret = 0;
 634        unsigned int comp_data_limit, decomp_data_limit;
 635        unsigned int comp_sg_limit, decomp_sg_limit;
 636        const struct maxsynccop_t {
 637                __be32 comp_elements;
 638                __be32 comp_data_limit;
 639                __be32 comp_sg_limit;
 640                __be32 decomp_elements;
 641                __be32 decomp_data_limit;
 642                __be32 decomp_sg_limit;
 643        } *maxsynccop;
 644
 645        if (prop->length != sizeof(*maxsynccop)) {
 646                dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
 647                dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
 648                                sizeof(*maxsynccop));
 649                ret = -EINVAL;
 650                goto out;
 651        }
 652
 653        maxsynccop = (const struct maxsynccop_t *)prop->value;
 654        comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
 655        comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
 656        decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
 657        decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
 658
 659        /* Use one limit rather than separate limits for compression and
 660         * decompression. Set a maximum for this so as not to exceed the
 661         * size that the header can support and round the value down to
 662         * the hardware page size (4K) */
 663        devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
 664
 665        devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
 666                                        65536);
 667
 668        if (devdata->max_sync_size < 4096) {
 669                dev_err(devdata->dev, "%s: hardware max data size (%u) is "
 670                                "less than the driver minimum, unable to use "
 671                                "the hardware device\n",
 672                                __func__, devdata->max_sync_size);
 673                ret = -EINVAL;
 674                goto out;
 675        }
 676
 677        nx842_pseries_constraints.maximum = devdata->max_sync_size;
 678
 679        devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
 680        if (devdata->max_sync_sg < 1) {
 681                dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
 682                                "less than the driver minimum, unable to use "
 683                                "the hardware device\n",
 684                                __func__, devdata->max_sync_sg);
 685                ret = -EINVAL;
 686                goto out;
 687        }
 688
 689out:
 690        return ret;
 691}
 692
 693/**
 694 *
 695 * nx842_OF_upd -- Handle OF properties updates for the device.
 696 *
 697 * Set all properties from the OF tree.  Optionally, a new property
 698 * can be provided by the @new_prop pointer to overwrite an existing value.
 699 * The device will remain disabled until all values are valid, this function
 700 * will return an error for updates unless all values are valid.
 701 *
 702 * @new_prop: If not NULL, this property is being updated.  If NULL, update
 703 *  all properties from the current values in the OF tree.
 704 *
 705 * Returns:
 706 *  0 - Success
 707 *  -ENOMEM - Could not allocate memory for new devdata structure
 708 *  -EINVAL - property value not found, new_prop is not a recognized
 709 *      property for the device or property value is not valid.
 710 *  -ENODEV - Device is not available
 711 */
 712static int nx842_OF_upd(struct property *new_prop)
 713{
 714        struct nx842_devdata *old_devdata = NULL;
 715        struct nx842_devdata *new_devdata = NULL;
 716        struct device_node *of_node = NULL;
 717        struct property *status = NULL;
 718        struct property *maxsglen = NULL;
 719        struct property *maxsyncop = NULL;
 720        int ret = 0;
 721        unsigned long flags;
 722
 723        new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 724        if (!new_devdata)
 725                return -ENOMEM;
 726
 727        spin_lock_irqsave(&devdata_mutex, flags);
 728        old_devdata = rcu_dereference_check(devdata,
 729                        lockdep_is_held(&devdata_mutex));
 730        if (old_devdata)
 731                of_node = old_devdata->dev->of_node;
 732
 733        if (!old_devdata || !of_node) {
 734                pr_err("%s: device is not available\n", __func__);
 735                spin_unlock_irqrestore(&devdata_mutex, flags);
 736                kfree(new_devdata);
 737                return -ENODEV;
 738        }
 739
 740        memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
 741        new_devdata->counters = old_devdata->counters;
 742
 743        /* Set ptrs for existing properties */
 744        status = of_find_property(of_node, "status", NULL);
 745        maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
 746        maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
 747        if (!status || !maxsglen || !maxsyncop) {
 748                dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
 749                ret = -EINVAL;
 750                goto error_out;
 751        }
 752
 753        /*
 754         * If this is a property update, there are only certain properties that
 755         * we care about. Bail if it isn't in the below list
 756         */
 757        if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
 758                         strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
 759                         strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
 760                goto out;
 761
 762        /* Perform property updates */
 763        ret = nx842_OF_upd_status(new_devdata, status);
 764        if (ret)
 765                goto error_out;
 766
 767        ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
 768        if (ret)
 769                goto error_out;
 770
 771        ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
 772        if (ret)
 773                goto error_out;
 774
 775out:
 776        dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
 777                        __func__, new_devdata->max_sync_size,
 778                        old_devdata->max_sync_size);
 779        dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
 780                        __func__, new_devdata->max_sync_sg,
 781                        old_devdata->max_sync_sg);
 782        dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
 783                        __func__, new_devdata->max_sg_len,
 784                        old_devdata->max_sg_len);
 785
 786        rcu_assign_pointer(devdata, new_devdata);
 787        spin_unlock_irqrestore(&devdata_mutex, flags);
 788        synchronize_rcu();
 789        dev_set_drvdata(new_devdata->dev, new_devdata);
 790        kfree(old_devdata);
 791        return 0;
 792
 793error_out:
 794        if (new_devdata) {
 795                dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
 796                nx842_OF_set_defaults(new_devdata);
 797                rcu_assign_pointer(devdata, new_devdata);
 798                spin_unlock_irqrestore(&devdata_mutex, flags);
 799                synchronize_rcu();
 800                dev_set_drvdata(new_devdata->dev, new_devdata);
 801                kfree(old_devdata);
 802        } else {
 803                dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
 804                spin_unlock_irqrestore(&devdata_mutex, flags);
 805        }
 806
 807        if (!ret)
 808                ret = -EINVAL;
 809        return ret;
 810}
 811
 812/**
 813 * nx842_OF_notifier - Process updates to OF properties for the device
 814 *
 815 * @np: notifier block
 816 * @action: notifier action
 817 * @update: struct pSeries_reconfig_prop_update pointer if action is
 818 *      PSERIES_UPDATE_PROPERTY
 819 *
 820 * Returns:
 821 *      NOTIFY_OK on success
 822 *      NOTIFY_BAD encoded with error number on failure, use
 823 *              notifier_to_errno() to decode this value
 824 */
 825static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
 826                             void *data)
 827{
 828        struct of_reconfig_data *upd = data;
 829        struct nx842_devdata *local_devdata;
 830        struct device_node *node = NULL;
 831
 832        rcu_read_lock();
 833        local_devdata = rcu_dereference(devdata);
 834        if (local_devdata)
 835                node = local_devdata->dev->of_node;
 836
 837        if (local_devdata &&
 838                        action == OF_RECONFIG_UPDATE_PROPERTY &&
 839                        !strcmp(upd->dn->name, node->name)) {
 840                rcu_read_unlock();
 841                nx842_OF_upd(upd->prop);
 842        } else
 843                rcu_read_unlock();
 844
 845        return NOTIFY_OK;
 846}
 847
 848static struct notifier_block nx842_of_nb = {
 849        .notifier_call = nx842_OF_notifier,
 850};
 851
 852#define nx842_counter_read(_name)                                       \
 853static ssize_t nx842_##_name##_show(struct device *dev,         \
 854                struct device_attribute *attr,                          \
 855                char *buf) {                                            \
 856        struct nx842_devdata *local_devdata;                    \
 857        int p = 0;                                                      \
 858        rcu_read_lock();                                                \
 859        local_devdata = rcu_dereference(devdata);                       \
 860        if (local_devdata)                                              \
 861                p = snprintf(buf, PAGE_SIZE, "%lld\n",                  \
 862                       atomic64_read(&local_devdata->counters->_name)); \
 863        rcu_read_unlock();                                              \
 864        return p;                                                       \
 865}
 866
 867#define NX842DEV_COUNTER_ATTR_RO(_name)                                 \
 868        nx842_counter_read(_name);                                      \
 869        static struct device_attribute dev_attr_##_name = __ATTR(_name, \
 870                                                0444,                   \
 871                                                nx842_##_name##_show,\
 872                                                NULL);
 873
 874NX842DEV_COUNTER_ATTR_RO(comp_complete);
 875NX842DEV_COUNTER_ATTR_RO(comp_failed);
 876NX842DEV_COUNTER_ATTR_RO(decomp_complete);
 877NX842DEV_COUNTER_ATTR_RO(decomp_failed);
 878NX842DEV_COUNTER_ATTR_RO(swdecomp);
 879
 880static ssize_t nx842_timehist_show(struct device *,
 881                struct device_attribute *, char *);
 882
 883static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
 884                nx842_timehist_show, NULL);
 885static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
 886                0444, nx842_timehist_show, NULL);
 887
 888static ssize_t nx842_timehist_show(struct device *dev,
 889                struct device_attribute *attr, char *buf) {
 890        char *p = buf;
 891        struct nx842_devdata *local_devdata;
 892        atomic64_t *times;
 893        int bytes_remain = PAGE_SIZE;
 894        int bytes;
 895        int i;
 896
 897        rcu_read_lock();
 898        local_devdata = rcu_dereference(devdata);
 899        if (!local_devdata) {
 900                rcu_read_unlock();
 901                return 0;
 902        }
 903
 904        if (attr == &dev_attr_comp_times)
 905                times = local_devdata->counters->comp_times;
 906        else if (attr == &dev_attr_decomp_times)
 907                times = local_devdata->counters->decomp_times;
 908        else {
 909                rcu_read_unlock();
 910                return 0;
 911        }
 912
 913        for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
 914                bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
 915                               i ? (2<<(i-1)) : 0, (2<<i)-1,
 916                               atomic64_read(&times[i]));
 917                bytes_remain -= bytes;
 918                p += bytes;
 919        }
 920        /* The last bucket holds everything over
 921         * 2<<(NX842_HIST_SLOTS - 2) us */
 922        bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
 923                        2<<(NX842_HIST_SLOTS - 2),
 924                        atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
 925        p += bytes;
 926
 927        rcu_read_unlock();
 928        return p - buf;
 929}
 930
 931static struct attribute *nx842_sysfs_entries[] = {
 932        &dev_attr_comp_complete.attr,
 933        &dev_attr_comp_failed.attr,
 934        &dev_attr_decomp_complete.attr,
 935        &dev_attr_decomp_failed.attr,
 936        &dev_attr_swdecomp.attr,
 937        &dev_attr_comp_times.attr,
 938        &dev_attr_decomp_times.attr,
 939        NULL,
 940};
 941
 942static struct attribute_group nx842_attribute_group = {
 943        .name = NULL,           /* put in device directory */
 944        .attrs = nx842_sysfs_entries,
 945};
 946
 947static struct nx842_driver nx842_pseries_driver = {
 948        .name =         KBUILD_MODNAME,
 949        .owner =        THIS_MODULE,
 950        .workmem_size = sizeof(struct nx842_workmem),
 951        .constraints =  &nx842_pseries_constraints,
 952        .compress =     nx842_pseries_compress,
 953        .decompress =   nx842_pseries_decompress,
 954};
 955
 956static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
 957{
 958        return nx842_crypto_init(tfm, &nx842_pseries_driver);
 959}
 960
 961static struct crypto_alg nx842_pseries_alg = {
 962        .cra_name               = "842",
 963        .cra_driver_name        = "842-nx",
 964        .cra_priority           = 300,
 965        .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 966        .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
 967        .cra_module             = THIS_MODULE,
 968        .cra_init               = nx842_pseries_crypto_init,
 969        .cra_exit               = nx842_crypto_exit,
 970        .cra_u                  = { .compress = {
 971        .coa_compress           = nx842_crypto_compress,
 972        .coa_decompress         = nx842_crypto_decompress } }
 973};
 974
 975static int nx842_probe(struct vio_dev *viodev,
 976                       const struct vio_device_id *id)
 977{
 978        struct nx842_devdata *old_devdata, *new_devdata = NULL;
 979        unsigned long flags;
 980        int ret = 0;
 981
 982        new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 983        if (!new_devdata)
 984                return -ENOMEM;
 985
 986        new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
 987                        GFP_NOFS);
 988        if (!new_devdata->counters) {
 989                kfree(new_devdata);
 990                return -ENOMEM;
 991        }
 992
 993        spin_lock_irqsave(&devdata_mutex, flags);
 994        old_devdata = rcu_dereference_check(devdata,
 995                        lockdep_is_held(&devdata_mutex));
 996
 997        if (old_devdata && old_devdata->vdev != NULL) {
 998                dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
 999                ret = -1;
1000                goto error_unlock;
1001        }
1002
1003        dev_set_drvdata(&viodev->dev, NULL);
1004
1005        new_devdata->vdev = viodev;
1006        new_devdata->dev = &viodev->dev;
1007        nx842_OF_set_defaults(new_devdata);
1008
1009        rcu_assign_pointer(devdata, new_devdata);
1010        spin_unlock_irqrestore(&devdata_mutex, flags);
1011        synchronize_rcu();
1012        kfree(old_devdata);
1013
1014        of_reconfig_notifier_register(&nx842_of_nb);
1015
1016        ret = nx842_OF_upd(NULL);
1017        if (ret)
1018                goto error;
1019
1020        ret = crypto_register_alg(&nx842_pseries_alg);
1021        if (ret) {
1022                dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1023                goto error;
1024        }
1025
1026        rcu_read_lock();
1027        dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1028        rcu_read_unlock();
1029
1030        if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1031                dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1032                ret = -1;
1033                goto error;
1034        }
1035
1036        return 0;
1037
1038error_unlock:
1039        spin_unlock_irqrestore(&devdata_mutex, flags);
1040        if (new_devdata)
1041                kfree(new_devdata->counters);
1042        kfree(new_devdata);
1043error:
1044        return ret;
1045}
1046
1047static void nx842_remove(struct vio_dev *viodev)
1048{
1049        struct nx842_devdata *old_devdata;
1050        unsigned long flags;
1051
1052        pr_info("Removing IBM Power 842 compression device\n");
1053        sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1054
1055        crypto_unregister_alg(&nx842_pseries_alg);
1056
1057        spin_lock_irqsave(&devdata_mutex, flags);
1058        old_devdata = rcu_dereference_check(devdata,
1059                        lockdep_is_held(&devdata_mutex));
1060        of_reconfig_notifier_unregister(&nx842_of_nb);
1061        RCU_INIT_POINTER(devdata, NULL);
1062        spin_unlock_irqrestore(&devdata_mutex, flags);
1063        synchronize_rcu();
1064        dev_set_drvdata(&viodev->dev, NULL);
1065        if (old_devdata)
1066                kfree(old_devdata->counters);
1067        kfree(old_devdata);
1068}
1069
1070static const struct vio_device_id nx842_vio_driver_ids[] = {
1071        {"ibm,compression-v1", "ibm,compression"},
1072        {"", ""},
1073};
1074MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1075
1076static struct vio_driver nx842_vio_driver = {
1077        .name = KBUILD_MODNAME,
1078        .probe = nx842_probe,
1079        .remove = nx842_remove,
1080        .get_desired_dma = nx842_get_desired_dma,
1081        .id_table = nx842_vio_driver_ids,
1082};
1083
1084static int __init nx842_pseries_init(void)
1085{
1086        struct nx842_devdata *new_devdata;
1087        int ret;
1088
1089        if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1090                return -ENODEV;
1091
1092        RCU_INIT_POINTER(devdata, NULL);
1093        new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1094        if (!new_devdata)
1095                return -ENOMEM;
1096
1097        RCU_INIT_POINTER(devdata, new_devdata);
1098
1099        ret = vio_register_driver(&nx842_vio_driver);
1100        if (ret) {
1101                pr_err("Could not register VIO driver %d\n", ret);
1102
1103                kfree(new_devdata);
1104                return ret;
1105        }
1106
1107        return 0;
1108}
1109
1110module_init(nx842_pseries_init);
1111
1112static void __exit nx842_pseries_exit(void)
1113{
1114        struct nx842_devdata *old_devdata;
1115        unsigned long flags;
1116
1117        crypto_unregister_alg(&nx842_pseries_alg);
1118
1119        spin_lock_irqsave(&devdata_mutex, flags);
1120        old_devdata = rcu_dereference_check(devdata,
1121                        lockdep_is_held(&devdata_mutex));
1122        RCU_INIT_POINTER(devdata, NULL);
1123        spin_unlock_irqrestore(&devdata_mutex, flags);
1124        synchronize_rcu();
1125        if (old_devdata && old_devdata->dev)
1126                dev_set_drvdata(old_devdata->dev, NULL);
1127        kfree(old_devdata);
1128        vio_unregister_driver(&nx842_vio_driver);
1129}
1130
1131module_exit(nx842_pseries_exit);
1132
1133