linux/drivers/crypto/hisilicon/hpre/hpre_main.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2018-2019 HiSilicon Limited. */
   3#include <linux/acpi.h>
   4#include <linux/aer.h>
   5#include <linux/bitops.h>
   6#include <linux/debugfs.h>
   7#include <linux/init.h>
   8#include <linux/io.h>
   9#include <linux/kernel.h>
  10#include <linux/module.h>
  11#include <linux/pci.h>
  12#include <linux/topology.h>
  13#include <linux/uacce.h>
  14#include "hpre.h"
  15
  16#define HPRE_QM_ABNML_INT_MASK          0x100004
  17#define HPRE_CTRL_CNT_CLR_CE_BIT        BIT(0)
  18#define HPRE_COMM_CNT_CLR_CE            0x0
  19#define HPRE_CTRL_CNT_CLR_CE            0x301000
  20#define HPRE_FSM_MAX_CNT                0x301008
  21#define HPRE_VFG_AXQOS                  0x30100c
  22#define HPRE_VFG_AXCACHE                0x301010
  23#define HPRE_RDCHN_INI_CFG              0x301014
  24#define HPRE_AWUSR_FP_CFG               0x301018
  25#define HPRE_BD_ENDIAN                  0x301020
  26#define HPRE_ECC_BYPASS                 0x301024
  27#define HPRE_RAS_WIDTH_CFG              0x301028
  28#define HPRE_POISON_BYPASS              0x30102c
  29#define HPRE_BD_ARUSR_CFG               0x301030
  30#define HPRE_BD_AWUSR_CFG               0x301034
  31#define HPRE_TYPES_ENB                  0x301038
  32#define HPRE_RSA_ENB                    BIT(0)
  33#define HPRE_ECC_ENB                    BIT(1)
  34#define HPRE_DATA_RUSER_CFG             0x30103c
  35#define HPRE_DATA_WUSER_CFG             0x301040
  36#define HPRE_INT_MASK                   0x301400
  37#define HPRE_INT_STATUS                 0x301800
  38#define HPRE_CORE_INT_ENABLE            0
  39#define HPRE_CORE_INT_DISABLE           0x003fffff
  40#define HPRE_RDCHN_INI_ST               0x301a00
  41#define HPRE_CLSTR_BASE                 0x302000
  42#define HPRE_CORE_EN_OFFSET             0x04
  43#define HPRE_CORE_INI_CFG_OFFSET        0x20
  44#define HPRE_CORE_INI_STATUS_OFFSET     0x80
  45#define HPRE_CORE_HTBT_WARN_OFFSET      0x8c
  46#define HPRE_CORE_IS_SCHD_OFFSET        0x90
  47
  48#define HPRE_RAS_CE_ENB                 0x301410
  49#define HPRE_HAC_RAS_CE_ENABLE          (BIT(0) | BIT(22) | BIT(23))
  50#define HPRE_RAS_NFE_ENB                0x301414
  51#define HPRE_HAC_RAS_NFE_ENABLE         0x3ffffe
  52#define HPRE_RAS_FE_ENB                 0x301418
  53#define HPRE_HAC_RAS_FE_ENABLE          0
  54
  55#define HPRE_CORE_ENB           (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET)
  56#define HPRE_CORE_INI_CFG       (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET)
  57#define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET)
  58#define HPRE_HAC_ECC1_CNT               0x301a04
  59#define HPRE_HAC_ECC2_CNT               0x301a08
  60#define HPRE_HAC_INT_STATUS             0x301800
  61#define HPRE_HAC_SOURCE_INT             0x301600
  62#define HPRE_CLSTR_ADDR_INTRVL          0x1000
  63#define HPRE_CLUSTER_INQURY             0x100
  64#define HPRE_CLSTR_ADDR_INQRY_RSLT      0x104
  65#define HPRE_TIMEOUT_ABNML_BIT          6
  66#define HPRE_PASID_EN_BIT               9
  67#define HPRE_REG_RD_INTVRL_US           10
  68#define HPRE_REG_RD_TMOUT_US            1000
  69#define HPRE_DBGFS_VAL_MAX_LEN          20
  70#define HPRE_PCI_DEVICE_ID              0xa258
  71#define HPRE_PCI_VF_DEVICE_ID           0xa259
  72#define HPRE_ADDR(qm, offset)           ((qm)->io_base + (offset))
  73#define HPRE_QM_USR_CFG_MASK            0xfffffffe
  74#define HPRE_QM_AXI_CFG_MASK            0xffff
  75#define HPRE_QM_VFG_AX_MASK             0xff
  76#define HPRE_BD_USR_MASK                0x3
  77#define HPRE_CLUSTER_CORE_MASK_V2       0xf
  78#define HPRE_CLUSTER_CORE_MASK_V3       0xff
  79
  80#define HPRE_AM_OOO_SHUTDOWN_ENB        0x301044
  81#define HPRE_AM_OOO_SHUTDOWN_ENABLE     BIT(0)
  82#define HPRE_WR_MSI_PORT                BIT(2)
  83
  84#define HPRE_CORE_ECC_2BIT_ERR          BIT(1)
  85#define HPRE_OOO_ECC_2BIT_ERR           BIT(5)
  86
  87#define HPRE_QM_BME_FLR                 BIT(7)
  88#define HPRE_QM_PM_FLR                  BIT(11)
  89#define HPRE_QM_SRIOV_FLR               BIT(12)
  90
  91#define HPRE_CLUSTERS_NUM(qm)           \
  92        (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : HPRE_CLUSTERS_NUM_V2)
  93#define HPRE_CLUSTER_CORE_MASK(qm)      \
  94        (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTER_CORE_MASK_V3 :\
  95                HPRE_CLUSTER_CORE_MASK_V2)
  96#define HPRE_VIA_MSI_DSM                1
  97#define HPRE_SQE_MASK_OFFSET            8
  98#define HPRE_SQE_MASK_LEN               24
  99
 100static const char hpre_name[] = "hisi_hpre";
 101static struct dentry *hpre_debugfs_root;
 102static const struct pci_device_id hpre_dev_ids[] = {
 103        { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_DEVICE_ID) },
 104        { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_VF_DEVICE_ID) },
 105        { 0, }
 106};
 107
 108MODULE_DEVICE_TABLE(pci, hpre_dev_ids);
 109
 110struct hpre_hw_error {
 111        u32 int_msk;
 112        const char *msg;
 113};
 114
 115static struct hisi_qm_list hpre_devices = {
 116        .register_to_crypto     = hpre_algs_register,
 117        .unregister_from_crypto = hpre_algs_unregister,
 118};
 119
 120static const char * const hpre_debug_file_name[] = {
 121        [HPRE_CLEAR_ENABLE] = "rdclr_en",
 122        [HPRE_CLUSTER_CTRL] = "cluster_ctrl",
 123};
 124
 125static const struct hpre_hw_error hpre_hw_errors[] = {
 126        { .int_msk = BIT(0), .msg = "core_ecc_1bit_err_int_set" },
 127        { .int_msk = BIT(1), .msg = "core_ecc_2bit_err_int_set" },
 128        { .int_msk = BIT(2), .msg = "dat_wb_poison_int_set" },
 129        { .int_msk = BIT(3), .msg = "dat_rd_poison_int_set" },
 130        { .int_msk = BIT(4), .msg = "bd_rd_poison_int_set" },
 131        { .int_msk = BIT(5), .msg = "ooo_ecc_2bit_err_int_set" },
 132        { .int_msk = BIT(6), .msg = "cluster1_shb_timeout_int_set" },
 133        { .int_msk = BIT(7), .msg = "cluster2_shb_timeout_int_set" },
 134        { .int_msk = BIT(8), .msg = "cluster3_shb_timeout_int_set" },
 135        { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" },
 136        { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" },
 137        { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" },
 138        { .int_msk = BIT(22), .msg = "pt_rng_timeout_int_set"},
 139        { .int_msk = BIT(23), .msg = "sva_fsm_timeout_int_set"},
 140        {
 141                /* sentinel */
 142        }
 143};
 144
 145static const u64 hpre_cluster_offsets[] = {
 146        [HPRE_CLUSTER0] =
 147                HPRE_CLSTR_BASE + HPRE_CLUSTER0 * HPRE_CLSTR_ADDR_INTRVL,
 148        [HPRE_CLUSTER1] =
 149                HPRE_CLSTR_BASE + HPRE_CLUSTER1 * HPRE_CLSTR_ADDR_INTRVL,
 150        [HPRE_CLUSTER2] =
 151                HPRE_CLSTR_BASE + HPRE_CLUSTER2 * HPRE_CLSTR_ADDR_INTRVL,
 152        [HPRE_CLUSTER3] =
 153                HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL,
 154};
 155
 156static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
 157        {"CORES_EN_STATUS          ",  HPRE_CORE_EN_OFFSET},
 158        {"CORES_INI_CFG              ",  HPRE_CORE_INI_CFG_OFFSET},
 159        {"CORES_INI_STATUS         ",  HPRE_CORE_INI_STATUS_OFFSET},
 160        {"CORES_HTBT_WARN         ",  HPRE_CORE_HTBT_WARN_OFFSET},
 161        {"CORES_IS_SCHD               ",  HPRE_CORE_IS_SCHD_OFFSET},
 162};
 163
 164static const struct debugfs_reg32 hpre_com_dfx_regs[] = {
 165        {"READ_CLR_EN          ",  HPRE_CTRL_CNT_CLR_CE},
 166        {"AXQOS                   ",  HPRE_VFG_AXQOS},
 167        {"AWUSR_CFG              ",  HPRE_AWUSR_FP_CFG},
 168        {"QM_ARUSR_MCFG1           ",  QM_ARUSER_M_CFG_1},
 169        {"QM_AWUSR_MCFG1           ",  QM_AWUSER_M_CFG_1},
 170        {"BD_ENDIAN               ",  HPRE_BD_ENDIAN},
 171        {"ECC_CHECK_CTRL       ",  HPRE_ECC_BYPASS},
 172        {"RAS_INT_WIDTH       ",  HPRE_RAS_WIDTH_CFG},
 173        {"POISON_BYPASS       ",  HPRE_POISON_BYPASS},
 174        {"BD_ARUSER               ",  HPRE_BD_ARUSR_CFG},
 175        {"BD_AWUSER               ",  HPRE_BD_AWUSR_CFG},
 176        {"DATA_ARUSER            ",  HPRE_DATA_RUSER_CFG},
 177        {"DATA_AWUSER           ",  HPRE_DATA_WUSER_CFG},
 178        {"INT_STATUS               ",  HPRE_INT_STATUS},
 179};
 180
 181static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = {
 182        "send_cnt",
 183        "recv_cnt",
 184        "send_fail_cnt",
 185        "send_busy_cnt",
 186        "over_thrhld_cnt",
 187        "overtime_thrhld",
 188        "invalid_req_cnt"
 189};
 190
 191static const struct kernel_param_ops hpre_uacce_mode_ops = {
 192        .set = uacce_mode_set,
 193        .get = param_get_int,
 194};
 195
 196/*
 197 * uacce_mode = 0 means hpre only register to crypto,
 198 * uacce_mode = 1 means hpre both register to crypto and uacce.
 199 */
 200static u32 uacce_mode = UACCE_MODE_NOUACCE;
 201module_param_cb(uacce_mode, &hpre_uacce_mode_ops, &uacce_mode, 0444);
 202MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
 203
 204static int pf_q_num_set(const char *val, const struct kernel_param *kp)
 205{
 206        return q_num_set(val, kp, HPRE_PCI_DEVICE_ID);
 207}
 208
 209static const struct kernel_param_ops hpre_pf_q_num_ops = {
 210        .set = pf_q_num_set,
 211        .get = param_get_int,
 212};
 213
 214static u32 pf_q_num = HPRE_PF_DEF_Q_NUM;
 215module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444);
 216MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)");
 217
 218static const struct kernel_param_ops vfs_num_ops = {
 219        .set = vfs_num_set,
 220        .get = param_get_int,
 221};
 222
 223static u32 vfs_num;
 224module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
 225MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
 226
 227struct hisi_qp *hpre_create_qp(u8 type)
 228{
 229        int node = cpu_to_node(smp_processor_id());
 230        struct hisi_qp *qp = NULL;
 231        int ret;
 232
 233        if (type != HPRE_V2_ALG_TYPE && type != HPRE_V3_ECC_ALG_TYPE)
 234                return NULL;
 235
 236        /*
 237         * type: 0 - RSA/DH. algorithm supported in V2,
 238         *       1 - ECC algorithm in V3.
 239         */
 240        ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp);
 241        if (!ret)
 242                return qp;
 243
 244        return NULL;
 245}
 246
 247static void hpre_config_pasid(struct hisi_qm *qm)
 248{
 249        u32 val1, val2;
 250
 251        if (qm->ver >= QM_HW_V3)
 252                return;
 253
 254        val1 = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG);
 255        val2 = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG);
 256        if (qm->use_sva) {
 257                val1 |= BIT(HPRE_PASID_EN_BIT);
 258                val2 |= BIT(HPRE_PASID_EN_BIT);
 259        } else {
 260                val1 &= ~BIT(HPRE_PASID_EN_BIT);
 261                val2 &= ~BIT(HPRE_PASID_EN_BIT);
 262        }
 263        writel_relaxed(val1, qm->io_base + HPRE_DATA_RUSER_CFG);
 264        writel_relaxed(val2, qm->io_base + HPRE_DATA_WUSER_CFG);
 265}
 266
 267static int hpre_cfg_by_dsm(struct hisi_qm *qm)
 268{
 269        struct device *dev = &qm->pdev->dev;
 270        union acpi_object *obj;
 271        guid_t guid;
 272
 273        if (guid_parse("b06b81ab-0134-4a45-9b0c-483447b95fa7", &guid)) {
 274                dev_err(dev, "Hpre GUID failed\n");
 275                return -EINVAL;
 276        }
 277
 278        /* Switch over to MSI handling due to non-standard PCI implementation */
 279        obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid,
 280                                0, HPRE_VIA_MSI_DSM, NULL);
 281        if (!obj) {
 282                dev_err(dev, "ACPI handle failed!\n");
 283                return -EIO;
 284        }
 285
 286        ACPI_FREE(obj);
 287
 288        return 0;
 289}
 290
 291static int hpre_set_cluster(struct hisi_qm *qm)
 292{
 293        u32 cluster_core_mask = HPRE_CLUSTER_CORE_MASK(qm);
 294        u8 clusters_num = HPRE_CLUSTERS_NUM(qm);
 295        struct device *dev = &qm->pdev->dev;
 296        unsigned long offset;
 297        u32 val = 0;
 298        int ret, i;
 299
 300        for (i = 0; i < clusters_num; i++) {
 301                offset = i * HPRE_CLSTR_ADDR_INTRVL;
 302
 303                /* clusters initiating */
 304                writel(cluster_core_mask,
 305                       HPRE_ADDR(qm, offset + HPRE_CORE_ENB));
 306                writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG));
 307                ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset +
 308                                        HPRE_CORE_INI_STATUS), val,
 309                                        ((val & cluster_core_mask) ==
 310                                        cluster_core_mask),
 311                                        HPRE_REG_RD_INTVRL_US,
 312                                        HPRE_REG_RD_TMOUT_US);
 313                if (ret) {
 314                        dev_err(dev,
 315                                "cluster %d int st status timeout!\n", i);
 316                        return -ETIMEDOUT;
 317                }
 318        }
 319
 320        return 0;
 321}
 322
 323/*
 324 * For Kunpeng 920, we should disable FLR triggered by hardware (BME/PM/SRIOV).
 325 * Or it may stay in D3 state when we bind and unbind hpre quickly,
 326 * as it does FLR triggered by hardware.
 327 */
 328static void disable_flr_of_bme(struct hisi_qm *qm)
 329{
 330        u32 val;
 331
 332        val = readl(HPRE_ADDR(qm, QM_PEH_AXUSER_CFG));
 333        val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR);
 334        val |= HPRE_QM_PM_FLR;
 335        writel(val, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG));
 336        writel(PEH_AXUSER_CFG_ENABLE, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG_ENABLE));
 337}
 338
 339static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
 340{
 341        struct device *dev = &qm->pdev->dev;
 342        u32 val;
 343        int ret;
 344
 345        writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE));
 346        writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE));
 347        writel_relaxed(HPRE_QM_AXI_CFG_MASK, HPRE_ADDR(qm, QM_AXI_M_CFG));
 348
 349        /* HPRE need more time, we close this interrupt */
 350        val = readl_relaxed(HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK));
 351        val |= BIT(HPRE_TIMEOUT_ABNML_BIT);
 352        writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK));
 353
 354        if (qm->ver >= QM_HW_V3)
 355                writel(HPRE_RSA_ENB | HPRE_ECC_ENB,
 356                        HPRE_ADDR(qm, HPRE_TYPES_ENB));
 357        else
 358                writel(HPRE_RSA_ENB, HPRE_ADDR(qm, HPRE_TYPES_ENB));
 359
 360        writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE));
 361        writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN));
 362        writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK));
 363        writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS));
 364        writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE));
 365        writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS));
 366
 367        writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_ARUSR_CFG));
 368        writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_AWUSR_CFG));
 369        writel(0x1, HPRE_ADDR(qm, HPRE_RDCHN_INI_CFG));
 370        ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, HPRE_RDCHN_INI_ST), val,
 371                                         val & BIT(0),
 372                        HPRE_REG_RD_INTVRL_US,
 373                        HPRE_REG_RD_TMOUT_US);
 374        if (ret) {
 375                dev_err(dev, "read rd channel timeout fail!\n");
 376                return -ETIMEDOUT;
 377        }
 378
 379        ret = hpre_set_cluster(qm);
 380        if (ret)
 381                return -ETIMEDOUT;
 382
 383        /* This setting is only needed by Kunpeng 920. */
 384        if (qm->ver == QM_HW_V2) {
 385                ret = hpre_cfg_by_dsm(qm);
 386                if (ret)
 387                        return ret;
 388
 389                disable_flr_of_bme(qm);
 390        }
 391
 392        /* Config data buffer pasid needed by Kunpeng 920 */
 393        hpre_config_pasid(qm);
 394
 395        return ret;
 396}
 397
 398static void hpre_cnt_regs_clear(struct hisi_qm *qm)
 399{
 400        u8 clusters_num = HPRE_CLUSTERS_NUM(qm);
 401        unsigned long offset;
 402        int i;
 403
 404        /* clear clusterX/cluster_ctrl */
 405        for (i = 0; i < clusters_num; i++) {
 406                offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL;
 407                writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY);
 408        }
 409
 410        /* clear rdclr_en */
 411        writel(0x0, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
 412
 413        hisi_qm_debug_regs_clear(qm);
 414}
 415
 416static void hpre_hw_error_disable(struct hisi_qm *qm)
 417{
 418        u32 val;
 419
 420        /* disable hpre hw error interrupts */
 421        writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK);
 422
 423        /* disable HPRE block master OOO when m-bit error occur */
 424        val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 425        val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
 426        writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 427}
 428
 429static void hpre_hw_error_enable(struct hisi_qm *qm)
 430{
 431        u32 val;
 432
 433        /* clear HPRE hw error source if having */
 434        writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT);
 435
 436        /* enable hpre hw error interrupts */
 437        writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK);
 438        writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB);
 439        writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB);
 440        writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB);
 441
 442        /* enable HPRE block master OOO when m-bit error occur */
 443        val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 444        val |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
 445        writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 446}
 447
 448static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
 449{
 450        struct hpre *hpre = container_of(file->debug, struct hpre, debug);
 451
 452        return &hpre->qm;
 453}
 454
 455static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file)
 456{
 457        struct hisi_qm *qm = hpre_file_to_qm(file);
 458
 459        return readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
 460               HPRE_CTRL_CNT_CLR_CE_BIT;
 461}
 462
 463static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val)
 464{
 465        struct hisi_qm *qm = hpre_file_to_qm(file);
 466        u32 tmp;
 467
 468        if (val != 1 && val != 0)
 469                return -EINVAL;
 470
 471        tmp = (readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
 472               ~HPRE_CTRL_CNT_CLR_CE_BIT) | val;
 473        writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
 474
 475        return 0;
 476}
 477
 478static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file)
 479{
 480        struct hisi_qm *qm = hpre_file_to_qm(file);
 481        int cluster_index = file->index - HPRE_CLUSTER_CTRL;
 482        unsigned long offset = HPRE_CLSTR_BASE +
 483                               cluster_index * HPRE_CLSTR_ADDR_INTRVL;
 484
 485        return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT);
 486}
 487
 488static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val)
 489{
 490        struct hisi_qm *qm = hpre_file_to_qm(file);
 491        int cluster_index = file->index - HPRE_CLUSTER_CTRL;
 492        unsigned long offset = HPRE_CLSTR_BASE + cluster_index *
 493                               HPRE_CLSTR_ADDR_INTRVL;
 494
 495        writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY);
 496
 497        return 0;
 498}
 499
 500static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf,
 501                                    size_t count, loff_t *pos)
 502{
 503        struct hpre_debugfs_file *file = filp->private_data;
 504        char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
 505        u32 val;
 506        int ret;
 507
 508        spin_lock_irq(&file->lock);
 509        switch (file->type) {
 510        case HPRE_CLEAR_ENABLE:
 511                val = hpre_clear_enable_read(file);
 512                break;
 513        case HPRE_CLUSTER_CTRL:
 514                val = hpre_cluster_inqry_read(file);
 515                break;
 516        default:
 517                spin_unlock_irq(&file->lock);
 518                return -EINVAL;
 519        }
 520        spin_unlock_irq(&file->lock);
 521        ret = snprintf(tbuf, HPRE_DBGFS_VAL_MAX_LEN, "%u\n", val);
 522        return simple_read_from_buffer(buf, count, pos, tbuf, ret);
 523}
 524
 525static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf,
 526                                     size_t count, loff_t *pos)
 527{
 528        struct hpre_debugfs_file *file = filp->private_data;
 529        char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
 530        unsigned long val;
 531        int len, ret;
 532
 533        if (*pos != 0)
 534                return 0;
 535
 536        if (count >= HPRE_DBGFS_VAL_MAX_LEN)
 537                return -ENOSPC;
 538
 539        len = simple_write_to_buffer(tbuf, HPRE_DBGFS_VAL_MAX_LEN - 1,
 540                                     pos, buf, count);
 541        if (len < 0)
 542                return len;
 543
 544        tbuf[len] = '\0';
 545        if (kstrtoul(tbuf, 0, &val))
 546                return -EFAULT;
 547
 548        spin_lock_irq(&file->lock);
 549        switch (file->type) {
 550        case HPRE_CLEAR_ENABLE:
 551                ret = hpre_clear_enable_write(file, val);
 552                if (ret)
 553                        goto err_input;
 554                break;
 555        case HPRE_CLUSTER_CTRL:
 556                ret = hpre_cluster_inqry_write(file, val);
 557                if (ret)
 558                        goto err_input;
 559                break;
 560        default:
 561                ret = -EINVAL;
 562                goto err_input;
 563        }
 564        spin_unlock_irq(&file->lock);
 565
 566        return count;
 567
 568err_input:
 569        spin_unlock_irq(&file->lock);
 570        return ret;
 571}
 572
 573static const struct file_operations hpre_ctrl_debug_fops = {
 574        .owner = THIS_MODULE,
 575        .open = simple_open,
 576        .read = hpre_ctrl_debug_read,
 577        .write = hpre_ctrl_debug_write,
 578};
 579
 580static int hpre_debugfs_atomic64_get(void *data, u64 *val)
 581{
 582        struct hpre_dfx *dfx_item = data;
 583
 584        *val = atomic64_read(&dfx_item->value);
 585
 586        return 0;
 587}
 588
 589static int hpre_debugfs_atomic64_set(void *data, u64 val)
 590{
 591        struct hpre_dfx *dfx_item = data;
 592        struct hpre_dfx *hpre_dfx = NULL;
 593
 594        if (dfx_item->type == HPRE_OVERTIME_THRHLD) {
 595                hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD;
 596                atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0);
 597        } else if (val) {
 598                return -EINVAL;
 599        }
 600
 601        atomic64_set(&dfx_item->value, val);
 602
 603        return 0;
 604}
 605
 606DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get,
 607                         hpre_debugfs_atomic64_set, "%llu\n");
 608
 609static int hpre_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
 610                                    enum hpre_ctrl_dbgfs_file type, int indx)
 611{
 612        struct hpre *hpre = container_of(qm, struct hpre, qm);
 613        struct hpre_debug *dbg = &hpre->debug;
 614        struct dentry *file_dir;
 615
 616        if (dir)
 617                file_dir = dir;
 618        else
 619                file_dir = qm->debug.debug_root;
 620
 621        if (type >= HPRE_DEBUG_FILE_NUM)
 622                return -EINVAL;
 623
 624        spin_lock_init(&dbg->files[indx].lock);
 625        dbg->files[indx].debug = dbg;
 626        dbg->files[indx].type = type;
 627        dbg->files[indx].index = indx;
 628        debugfs_create_file(hpre_debug_file_name[type], 0600, file_dir,
 629                            dbg->files + indx, &hpre_ctrl_debug_fops);
 630
 631        return 0;
 632}
 633
 634static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm)
 635{
 636        struct device *dev = &qm->pdev->dev;
 637        struct debugfs_regset32 *regset;
 638
 639        regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 640        if (!regset)
 641                return -ENOMEM;
 642
 643        regset->regs = hpre_com_dfx_regs;
 644        regset->nregs = ARRAY_SIZE(hpre_com_dfx_regs);
 645        regset->base = qm->io_base;
 646
 647        debugfs_create_regset32("regs", 0444,  qm->debug.debug_root, regset);
 648        return 0;
 649}
 650
 651static int hpre_cluster_debugfs_init(struct hisi_qm *qm)
 652{
 653        u8 clusters_num = HPRE_CLUSTERS_NUM(qm);
 654        struct device *dev = &qm->pdev->dev;
 655        char buf[HPRE_DBGFS_VAL_MAX_LEN];
 656        struct debugfs_regset32 *regset;
 657        struct dentry *tmp_d;
 658        int i, ret;
 659
 660        for (i = 0; i < clusters_num; i++) {
 661                ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i);
 662                if (ret < 0)
 663                        return -EINVAL;
 664                tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
 665
 666                regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 667                if (!regset)
 668                        return -ENOMEM;
 669
 670                regset->regs = hpre_cluster_dfx_regs;
 671                regset->nregs = ARRAY_SIZE(hpre_cluster_dfx_regs);
 672                regset->base = qm->io_base + hpre_cluster_offsets[i];
 673
 674                debugfs_create_regset32("regs", 0444, tmp_d, regset);
 675                ret = hpre_create_debugfs_file(qm, tmp_d, HPRE_CLUSTER_CTRL,
 676                                               i + HPRE_CLUSTER_CTRL);
 677                if (ret)
 678                        return ret;
 679        }
 680
 681        return 0;
 682}
 683
 684static int hpre_ctrl_debug_init(struct hisi_qm *qm)
 685{
 686        int ret;
 687
 688        ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE,
 689                                       HPRE_CLEAR_ENABLE);
 690        if (ret)
 691                return ret;
 692
 693        ret = hpre_pf_comm_regs_debugfs_init(qm);
 694        if (ret)
 695                return ret;
 696
 697        return hpre_cluster_debugfs_init(qm);
 698}
 699
 700static void hpre_dfx_debug_init(struct hisi_qm *qm)
 701{
 702        struct hpre *hpre = container_of(qm, struct hpre, qm);
 703        struct hpre_dfx *dfx = hpre->debug.dfx;
 704        struct dentry *parent;
 705        int i;
 706
 707        parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root);
 708        for (i = 0; i < HPRE_DFX_FILE_NUM; i++) {
 709                dfx[i].type = i;
 710                debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i],
 711                                    &hpre_atomic64_ops);
 712        }
 713}
 714
 715static int hpre_debugfs_init(struct hisi_qm *qm)
 716{
 717        struct device *dev = &qm->pdev->dev;
 718        int ret;
 719
 720        qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
 721                                                  hpre_debugfs_root);
 722
 723        qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET;
 724        qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN;
 725        hisi_qm_debug_init(qm);
 726
 727        if (qm->pdev->device == HPRE_PCI_DEVICE_ID) {
 728                ret = hpre_ctrl_debug_init(qm);
 729                if (ret)
 730                        goto failed_to_create;
 731        }
 732
 733        hpre_dfx_debug_init(qm);
 734
 735        return 0;
 736
 737failed_to_create:
 738        debugfs_remove_recursive(qm->debug.debug_root);
 739        return ret;
 740}
 741
 742static void hpre_debugfs_exit(struct hisi_qm *qm)
 743{
 744        debugfs_remove_recursive(qm->debug.debug_root);
 745}
 746
 747static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
 748{
 749        if (pdev->revision == QM_HW_V1) {
 750                pci_warn(pdev, "HPRE version 1 is not supported!\n");
 751                return -EINVAL;
 752        }
 753
 754        if (pdev->revision >= QM_HW_V3)
 755                qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2";
 756        else
 757                qm->algs = "rsa\ndh";
 758        qm->mode = uacce_mode;
 759        qm->pdev = pdev;
 760        qm->ver = pdev->revision;
 761        qm->sqe_size = HPRE_SQE_SIZE;
 762        qm->dev_name = hpre_name;
 763
 764        qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ?
 765                        QM_HW_PF : QM_HW_VF;
 766        if (qm->fun_type == QM_HW_PF) {
 767                qm->qp_base = HPRE_PF_DEF_Q_BASE;
 768                qm->qp_num = pf_q_num;
 769                qm->debug.curr_qm_qp_num = pf_q_num;
 770                qm->qm_list = &hpre_devices;
 771        }
 772
 773        return hisi_qm_init(qm);
 774}
 775
 776static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
 777{
 778        const struct hpre_hw_error *err = hpre_hw_errors;
 779        struct device *dev = &qm->pdev->dev;
 780
 781        while (err->msg) {
 782                if (err->int_msk & err_sts)
 783                        dev_warn(dev, "%s [error status=0x%x] found\n",
 784                                 err->msg, err->int_msk);
 785                err++;
 786        }
 787}
 788
 789static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
 790{
 791        return readl(qm->io_base + HPRE_HAC_INT_STATUS);
 792}
 793
 794static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
 795{
 796        writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT);
 797}
 798
 799static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
 800{
 801        u32 value;
 802
 803        value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 804        writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE,
 805               HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
 806        writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE,
 807               HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
 808}
 809
 810static void hpre_err_info_init(struct hisi_qm *qm)
 811{
 812        struct hisi_qm_err_info *err_info = &qm->err_info;
 813
 814        err_info->ce = QM_BASE_CE;
 815        err_info->fe = 0;
 816        err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR |
 817                                   HPRE_OOO_ECC_2BIT_ERR;
 818        err_info->dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE;
 819        err_info->msi_wr_port = HPRE_WR_MSI_PORT;
 820        err_info->acpi_rst = "HRST";
 821        err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT;
 822}
 823
 824static const struct hisi_qm_err_ini hpre_err_ini = {
 825        .hw_init                = hpre_set_user_domain_and_cache,
 826        .hw_err_enable          = hpre_hw_error_enable,
 827        .hw_err_disable         = hpre_hw_error_disable,
 828        .get_dev_hw_err_status  = hpre_get_hw_err_status,
 829        .clear_dev_hw_err_status = hpre_clear_hw_err_status,
 830        .log_dev_hw_err         = hpre_log_hw_error,
 831        .open_axi_master_ooo    = hpre_open_axi_master_ooo,
 832        .err_info_init          = hpre_err_info_init,
 833};
 834
 835static int hpre_pf_probe_init(struct hpre *hpre)
 836{
 837        struct hisi_qm *qm = &hpre->qm;
 838        int ret;
 839
 840        ret = hpre_set_user_domain_and_cache(qm);
 841        if (ret)
 842                return ret;
 843
 844        qm->err_ini = &hpre_err_ini;
 845        qm->err_ini->err_info_init(qm);
 846        hisi_qm_dev_err_init(qm);
 847
 848        return 0;
 849}
 850
 851static int hpre_probe_init(struct hpre *hpre)
 852{
 853        struct hisi_qm *qm = &hpre->qm;
 854        int ret;
 855
 856        if (qm->fun_type == QM_HW_PF) {
 857                ret = hpre_pf_probe_init(hpre);
 858                if (ret)
 859                        return ret;
 860        }
 861
 862        return 0;
 863}
 864
 865static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 866{
 867        struct hisi_qm *qm;
 868        struct hpre *hpre;
 869        int ret;
 870
 871        hpre = devm_kzalloc(&pdev->dev, sizeof(*hpre), GFP_KERNEL);
 872        if (!hpre)
 873                return -ENOMEM;
 874
 875        qm = &hpre->qm;
 876        ret = hpre_qm_init(qm, pdev);
 877        if (ret) {
 878                pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret);
 879                return ret;
 880        }
 881
 882        ret = hpre_probe_init(hpre);
 883        if (ret) {
 884                pci_err(pdev, "Failed to probe (%d)!\n", ret);
 885                goto err_with_qm_init;
 886        }
 887
 888        ret = hisi_qm_start(qm);
 889        if (ret)
 890                goto err_with_err_init;
 891
 892        ret = hpre_debugfs_init(qm);
 893        if (ret)
 894                dev_warn(&pdev->dev, "init debugfs fail!\n");
 895
 896        ret = hisi_qm_alg_register(qm, &hpre_devices);
 897        if (ret < 0) {
 898                pci_err(pdev, "fail to register algs to crypto!\n");
 899                goto err_with_qm_start;
 900        }
 901
 902        if (qm->uacce) {
 903                ret = uacce_register(qm->uacce);
 904                if (ret) {
 905                        pci_err(pdev, "failed to register uacce (%d)!\n", ret);
 906                        goto err_with_alg_register;
 907                }
 908        }
 909
 910        if (qm->fun_type == QM_HW_PF && vfs_num) {
 911                ret = hisi_qm_sriov_enable(pdev, vfs_num);
 912                if (ret < 0)
 913                        goto err_with_alg_register;
 914        }
 915
 916        return 0;
 917
 918err_with_alg_register:
 919        hisi_qm_alg_unregister(qm, &hpre_devices);
 920
 921err_with_qm_start:
 922        hpre_debugfs_exit(qm);
 923        hisi_qm_stop(qm, QM_NORMAL);
 924
 925err_with_err_init:
 926        hisi_qm_dev_err_uninit(qm);
 927
 928err_with_qm_init:
 929        hisi_qm_uninit(qm);
 930
 931        return ret;
 932}
 933
 934static void hpre_remove(struct pci_dev *pdev)
 935{
 936        struct hisi_qm *qm = pci_get_drvdata(pdev);
 937        int ret;
 938
 939        hisi_qm_wait_task_finish(qm, &hpre_devices);
 940        hisi_qm_alg_unregister(qm, &hpre_devices);
 941        if (qm->fun_type == QM_HW_PF && qm->vfs_num) {
 942                ret = hisi_qm_sriov_disable(pdev, true);
 943                if (ret) {
 944                        pci_err(pdev, "Disable SRIOV fail!\n");
 945                        return;
 946                }
 947        }
 948
 949        hpre_debugfs_exit(qm);
 950        hisi_qm_stop(qm, QM_NORMAL);
 951
 952        if (qm->fun_type == QM_HW_PF) {
 953                hpre_cnt_regs_clear(qm);
 954                qm->debug.curr_qm_qp_num = 0;
 955                hisi_qm_dev_err_uninit(qm);
 956        }
 957
 958        hisi_qm_uninit(qm);
 959}
 960
 961static const struct pci_error_handlers hpre_err_handler = {
 962        .error_detected         = hisi_qm_dev_err_detected,
 963        .slot_reset             = hisi_qm_dev_slot_reset,
 964        .reset_prepare          = hisi_qm_reset_prepare,
 965        .reset_done             = hisi_qm_reset_done,
 966};
 967
 968static struct pci_driver hpre_pci_driver = {
 969        .name                   = hpre_name,
 970        .id_table               = hpre_dev_ids,
 971        .probe                  = hpre_probe,
 972        .remove                 = hpre_remove,
 973        .sriov_configure        = IS_ENABLED(CONFIG_PCI_IOV) ?
 974                                  hisi_qm_sriov_configure : NULL,
 975        .err_handler            = &hpre_err_handler,
 976        .shutdown               = hisi_qm_dev_shutdown,
 977};
 978
 979static void hpre_register_debugfs(void)
 980{
 981        if (!debugfs_initialized())
 982                return;
 983
 984        hpre_debugfs_root = debugfs_create_dir(hpre_name, NULL);
 985}
 986
 987static void hpre_unregister_debugfs(void)
 988{
 989        debugfs_remove_recursive(hpre_debugfs_root);
 990}
 991
 992static int __init hpre_init(void)
 993{
 994        int ret;
 995
 996        hisi_qm_init_list(&hpre_devices);
 997        hpre_register_debugfs();
 998
 999        ret = pci_register_driver(&hpre_pci_driver);
1000        if (ret) {
1001                hpre_unregister_debugfs();
1002                pr_err("hpre: can't register hisi hpre driver.\n");
1003        }
1004
1005        return ret;
1006}
1007
1008static void __exit hpre_exit(void)
1009{
1010        pci_unregister_driver(&hpre_pci_driver);
1011        hpre_unregister_debugfs();
1012}
1013
1014module_init(hpre_init);
1015module_exit(hpre_exit);
1016
1017MODULE_LICENSE("GPL v2");
1018MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
1019MODULE_AUTHOR("Meng Yu <yumeng18@huawei.com>");
1020MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator");
1021