linux/drivers/crypto/hisilicon/qm.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2019 HiSilicon Limited. */
   3#include <asm/page.h>
   4#include <linux/acpi.h>
   5#include <linux/aer.h>
   6#include <linux/bitmap.h>
   7#include <linux/debugfs.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/idr.h>
  10#include <linux/io.h>
  11#include <linux/irqreturn.h>
  12#include <linux/log2.h>
  13#include <linux/seq_file.h>
  14#include <linux/slab.h>
  15#include <linux/uacce.h>
  16#include <linux/uaccess.h>
  17#include <uapi/misc/uacce/hisi_qm.h>
  18#include "qm.h"
  19
  20/* eq/aeq irq enable */
  21#define QM_VF_AEQ_INT_SOURCE            0x0
  22#define QM_VF_AEQ_INT_MASK              0x4
  23#define QM_VF_EQ_INT_SOURCE             0x8
  24#define QM_VF_EQ_INT_MASK               0xc
  25#define QM_IRQ_NUM_V1                   1
  26#define QM_IRQ_NUM_PF_V2                4
  27#define QM_IRQ_NUM_VF_V2                2
  28
  29#define QM_EQ_EVENT_IRQ_VECTOR          0
  30#define QM_AEQ_EVENT_IRQ_VECTOR         1
  31#define QM_ABNORMAL_EVENT_IRQ_VECTOR    3
  32
  33/* mailbox */
  34#define QM_MB_CMD_SQC                   0x0
  35#define QM_MB_CMD_CQC                   0x1
  36#define QM_MB_CMD_EQC                   0x2
  37#define QM_MB_CMD_AEQC                  0x3
  38#define QM_MB_CMD_SQC_BT                0x4
  39#define QM_MB_CMD_CQC_BT                0x5
  40#define QM_MB_CMD_SQC_VFT_V2            0x6
  41#define QM_MB_CMD_STOP_QP               0x8
  42
  43#define QM_MB_CMD_SEND_BASE             0x300
  44#define QM_MB_EVENT_SHIFT               8
  45#define QM_MB_BUSY_SHIFT                13
  46#define QM_MB_OP_SHIFT                  14
  47#define QM_MB_CMD_DATA_ADDR_L           0x304
  48#define QM_MB_CMD_DATA_ADDR_H           0x308
  49
  50/* sqc shift */
  51#define QM_SQ_HOP_NUM_SHIFT             0
  52#define QM_SQ_PAGE_SIZE_SHIFT           4
  53#define QM_SQ_BUF_SIZE_SHIFT            8
  54#define QM_SQ_SQE_SIZE_SHIFT            12
  55#define QM_SQ_PRIORITY_SHIFT            0
  56#define QM_SQ_ORDERS_SHIFT              4
  57#define QM_SQ_TYPE_SHIFT                8
  58#define QM_QC_PASID_ENABLE              0x1
  59#define QM_QC_PASID_ENABLE_SHIFT        7
  60
  61#define QM_SQ_TYPE_MASK                 GENMASK(3, 0)
  62#define QM_SQ_TAIL_IDX(sqc)             ((le16_to_cpu((sqc)->w11) >> 6) & 0x1)
  63
  64/* cqc shift */
  65#define QM_CQ_HOP_NUM_SHIFT             0
  66#define QM_CQ_PAGE_SIZE_SHIFT           4
  67#define QM_CQ_BUF_SIZE_SHIFT            8
  68#define QM_CQ_CQE_SIZE_SHIFT            12
  69#define QM_CQ_PHASE_SHIFT               0
  70#define QM_CQ_FLAG_SHIFT                1
  71
  72#define QM_CQE_PHASE(cqe)               (le16_to_cpu((cqe)->w7) & 0x1)
  73#define QM_QC_CQE_SIZE                  4
  74#define QM_CQ_TAIL_IDX(cqc)             ((le16_to_cpu((cqc)->w11) >> 6) & 0x1)
  75
  76/* eqc shift */
  77#define QM_EQE_AEQE_SIZE                (2UL << 12)
  78#define QM_EQC_PHASE_SHIFT              16
  79
  80#define QM_EQE_PHASE(eqe)               ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
  81#define QM_EQE_CQN_MASK                 GENMASK(15, 0)
  82
  83#define QM_AEQE_PHASE(aeqe)             ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
  84#define QM_AEQE_TYPE_SHIFT              17
  85
  86#define QM_DOORBELL_CMD_SQ              0
  87#define QM_DOORBELL_CMD_CQ              1
  88#define QM_DOORBELL_CMD_EQ              2
  89#define QM_DOORBELL_CMD_AEQ             3
  90
  91#define QM_DOORBELL_BASE_V1             0x340
  92#define QM_DB_CMD_SHIFT_V1              16
  93#define QM_DB_INDEX_SHIFT_V1            32
  94#define QM_DB_PRIORITY_SHIFT_V1         48
  95#define QM_DOORBELL_SQ_CQ_BASE_V2       0x1000
  96#define QM_DOORBELL_EQ_AEQ_BASE_V2      0x2000
  97#define QM_QUE_ISO_CFG_V                0x0030
  98#define QM_QUE_ISO_EN                   0x100154
  99#define QM_CAPBILITY                    0x100158
 100#define QM_QP_NUN_MASK                  GENMASK(10, 0)
 101#define QM_QP_DB_INTERVAL               0x10000
 102#define QM_QP_MAX_NUM_SHIFT             11
 103#define QM_DB_CMD_SHIFT_V2              12
 104#define QM_DB_RAND_SHIFT_V2             16
 105#define QM_DB_INDEX_SHIFT_V2            32
 106#define QM_DB_PRIORITY_SHIFT_V2         48
 107
 108#define QM_MEM_START_INIT               0x100040
 109#define QM_MEM_INIT_DONE                0x100044
 110#define QM_VFT_CFG_RDY                  0x10006c
 111#define QM_VFT_CFG_OP_WR                0x100058
 112#define QM_VFT_CFG_TYPE                 0x10005c
 113#define QM_SQC_VFT                      0x0
 114#define QM_CQC_VFT                      0x1
 115#define QM_VFT_CFG                      0x100060
 116#define QM_VFT_CFG_OP_ENABLE            0x100054
 117
 118#define QM_VFT_CFG_DATA_L               0x100064
 119#define QM_VFT_CFG_DATA_H               0x100068
 120#define QM_SQC_VFT_BUF_SIZE             (7ULL << 8)
 121#define QM_SQC_VFT_SQC_SIZE             (5ULL << 12)
 122#define QM_SQC_VFT_INDEX_NUMBER         (1ULL << 16)
 123#define QM_SQC_VFT_START_SQN_SHIFT      28
 124#define QM_SQC_VFT_VALID                (1ULL << 44)
 125#define QM_SQC_VFT_SQN_SHIFT            45
 126#define QM_CQC_VFT_BUF_SIZE             (7ULL << 8)
 127#define QM_CQC_VFT_SQC_SIZE             (5ULL << 12)
 128#define QM_CQC_VFT_INDEX_NUMBER         (1ULL << 16)
 129#define QM_CQC_VFT_VALID                (1ULL << 28)
 130
 131#define QM_SQC_VFT_BASE_SHIFT_V2        28
 132#define QM_SQC_VFT_BASE_MASK_V2         GENMASK(15, 0)
 133#define QM_SQC_VFT_NUM_SHIFT_V2         45
 134#define QM_SQC_VFT_NUM_MASK_v2          GENMASK(9, 0)
 135
 136#define QM_DFX_CNT_CLR_CE               0x100118
 137
 138#define QM_ABNORMAL_INT_SOURCE          0x100000
 139#define QM_ABNORMAL_INT_SOURCE_CLR      GENMASK(14, 0)
 140#define QM_ABNORMAL_INT_MASK            0x100004
 141#define QM_ABNORMAL_INT_MASK_VALUE      0x7fff
 142#define QM_ABNORMAL_INT_STATUS          0x100008
 143#define QM_ABNORMAL_INT_SET             0x10000c
 144#define QM_ABNORMAL_INF00               0x100010
 145#define QM_FIFO_OVERFLOW_TYPE           0xc0
 146#define QM_FIFO_OVERFLOW_TYPE_SHIFT     6
 147#define QM_FIFO_OVERFLOW_VF             0x3f
 148#define QM_ABNORMAL_INF01               0x100014
 149#define QM_DB_TIMEOUT_TYPE              0xc0
 150#define QM_DB_TIMEOUT_TYPE_SHIFT        6
 151#define QM_DB_TIMEOUT_VF                0x3f
 152#define QM_RAS_CE_ENABLE                0x1000ec
 153#define QM_RAS_FE_ENABLE                0x1000f0
 154#define QM_RAS_NFE_ENABLE               0x1000f4
 155#define QM_RAS_CE_THRESHOLD             0x1000f8
 156#define QM_RAS_CE_TIMES_PER_IRQ         1
 157#define QM_RAS_MSI_INT_SEL              0x1040f4
 158
 159#define QM_RESET_WAIT_TIMEOUT           400
 160#define QM_PEH_VENDOR_ID                0x1000d8
 161#define ACC_VENDOR_ID_VALUE             0x5a5a
 162#define QM_PEH_DFX_INFO0                0x1000fc
 163#define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3
 164#define ACC_PEH_MSI_DISABLE             GENMASK(31, 0)
 165#define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1
 166#define ACC_MASTER_TRANS_RETURN_RW      3
 167#define ACC_MASTER_TRANS_RETURN         0x300150
 168#define ACC_MASTER_GLOBAL_CTRL          0x300000
 169#define ACC_AM_CFG_PORT_WR_EN           0x30001c
 170#define QM_RAS_NFE_MBIT_DISABLE         ~QM_ECC_MBIT
 171#define ACC_AM_ROB_ECC_INT_STS          0x300104
 172#define ACC_ROB_ECC_ERR_MULTPL          BIT(1)
 173
 174#define QM_DFX_MB_CNT_VF                0x104010
 175#define QM_DFX_DB_CNT_VF                0x104020
 176#define QM_DFX_SQE_CNT_VF_SQN           0x104030
 177#define QM_DFX_CQE_CNT_VF_CQN           0x104040
 178#define QM_DFX_QN_SHIFT                 16
 179#define CURRENT_FUN_MASK                GENMASK(5, 0)
 180#define CURRENT_Q_MASK                  GENMASK(31, 16)
 181
 182#define POLL_PERIOD                     10
 183#define POLL_TIMEOUT                    1000
 184#define WAIT_PERIOD_US_MAX              200
 185#define WAIT_PERIOD_US_MIN              100
 186#define MAX_WAIT_COUNTS                 1000
 187#define QM_CACHE_WB_START               0x204
 188#define QM_CACHE_WB_DONE                0x208
 189
 190#define PCI_BAR_2                       2
 191#define PCI_BAR_4                       4
 192#define QM_SQE_DATA_ALIGN_MASK          GENMASK(6, 0)
 193#define QMC_ALIGN(sz)                   ALIGN(sz, 32)
 194
 195#define QM_DBG_READ_LEN         256
 196#define QM_DBG_WRITE_LEN                1024
 197#define QM_DBG_TMP_BUF_LEN              22
 198#define QM_PCI_COMMAND_INVALID          ~0
 199
 200#define WAIT_PERIOD                     20
 201#define REMOVE_WAIT_DELAY               10
 202#define QM_SQE_ADDR_MASK                GENMASK(7, 0)
 203#define QM_EQ_DEPTH                     (1024 * 2)
 204
 205#define QM_DRIVER_REMOVING              0
 206#define QM_RST_SCHED                    1
 207#define QM_RESETTING                    2
 208
 209#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
 210        (((hop_num) << QM_CQ_HOP_NUM_SHIFT)     | \
 211        ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT)      | \
 212        ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT)      | \
 213        ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 214
 215#define QM_MK_CQC_DW3_V2(cqe_sz) \
 216        ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 217
 218#define QM_MK_SQC_W13(priority, orders, alg_type) \
 219        (((priority) << QM_SQ_PRIORITY_SHIFT)   | \
 220        ((orders) << QM_SQ_ORDERS_SHIFT)        | \
 221        (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
 222
 223#define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
 224        (((hop_num) << QM_SQ_HOP_NUM_SHIFT)     | \
 225        ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT)      | \
 226        ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT)      | \
 227        ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 228
 229#define QM_MK_SQC_DW3_V2(sqe_sz) \
 230        ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 231
 232#define INIT_QC_COMMON(qc, base, pasid) do {                    \
 233        (qc)->head = 0;                                         \
 234        (qc)->tail = 0;                                         \
 235        (qc)->base_l = cpu_to_le32(lower_32_bits(base));        \
 236        (qc)->base_h = cpu_to_le32(upper_32_bits(base));        \
 237        (qc)->dw3 = 0;                                          \
 238        (qc)->w8 = 0;                                           \
 239        (qc)->rsvd0 = 0;                                        \
 240        (qc)->pasid = cpu_to_le16(pasid);                       \
 241        (qc)->w11 = 0;                                          \
 242        (qc)->rsvd1 = 0;                                        \
 243} while (0)
 244
 245enum vft_type {
 246        SQC_VFT = 0,
 247        CQC_VFT,
 248};
 249
 250enum acc_err_result {
 251        ACC_ERR_NONE,
 252        ACC_ERR_NEED_RESET,
 253        ACC_ERR_RECOVERED,
 254};
 255
 256struct qm_cqe {
 257        __le32 rsvd0;
 258        __le16 cmd_id;
 259        __le16 rsvd1;
 260        __le16 sq_head;
 261        __le16 sq_num;
 262        __le16 rsvd2;
 263        __le16 w7;
 264};
 265
 266struct qm_eqe {
 267        __le32 dw0;
 268};
 269
 270struct qm_aeqe {
 271        __le32 dw0;
 272};
 273
 274struct qm_sqc {
 275        __le16 head;
 276        __le16 tail;
 277        __le32 base_l;
 278        __le32 base_h;
 279        __le32 dw3;
 280        __le16 w8;
 281        __le16 rsvd0;
 282        __le16 pasid;
 283        __le16 w11;
 284        __le16 cq_num;
 285        __le16 w13;
 286        __le32 rsvd1;
 287};
 288
 289struct qm_cqc {
 290        __le16 head;
 291        __le16 tail;
 292        __le32 base_l;
 293        __le32 base_h;
 294        __le32 dw3;
 295        __le16 w8;
 296        __le16 rsvd0;
 297        __le16 pasid;
 298        __le16 w11;
 299        __le32 dw6;
 300        __le32 rsvd1;
 301};
 302
 303struct qm_eqc {
 304        __le16 head;
 305        __le16 tail;
 306        __le32 base_l;
 307        __le32 base_h;
 308        __le32 dw3;
 309        __le32 rsvd[2];
 310        __le32 dw6;
 311};
 312
 313struct qm_aeqc {
 314        __le16 head;
 315        __le16 tail;
 316        __le32 base_l;
 317        __le32 base_h;
 318        __le32 dw3;
 319        __le32 rsvd[2];
 320        __le32 dw6;
 321};
 322
 323struct qm_mailbox {
 324        __le16 w0;
 325        __le16 queue_num;
 326        __le32 base_l;
 327        __le32 base_h;
 328        __le32 rsvd;
 329};
 330
 331struct qm_doorbell {
 332        __le16 queue_num;
 333        __le16 cmd;
 334        __le16 index;
 335        __le16 priority;
 336};
 337
 338struct hisi_qm_resource {
 339        struct hisi_qm *qm;
 340        int distance;
 341        struct list_head list;
 342};
 343
 344struct hisi_qm_hw_ops {
 345        int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
 346        void (*qm_db)(struct hisi_qm *qm, u16 qn,
 347                      u8 cmd, u16 index, u8 priority);
 348        u32 (*get_irq_num)(struct hisi_qm *qm);
 349        int (*debug_init)(struct hisi_qm *qm);
 350        void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe);
 351        void (*hw_error_uninit)(struct hisi_qm *qm);
 352        enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
 353        int (*stop_qp)(struct hisi_qp *qp);
 354};
 355
 356struct qm_dfx_item {
 357        const char *name;
 358        u32 offset;
 359};
 360
 361static struct qm_dfx_item qm_dfx_files[] = {
 362        {"err_irq", offsetof(struct qm_dfx, err_irq_cnt)},
 363        {"aeq_irq", offsetof(struct qm_dfx, aeq_irq_cnt)},
 364        {"abnormal_irq", offsetof(struct qm_dfx, abnormal_irq_cnt)},
 365        {"create_qp_err", offsetof(struct qm_dfx, create_qp_err_cnt)},
 366        {"mb_err", offsetof(struct qm_dfx, mb_err_cnt)},
 367};
 368
 369static const char * const qm_debug_file_name[] = {
 370        [CURRENT_QM]   = "current_qm",
 371        [CURRENT_Q]    = "current_q",
 372        [CLEAR_ENABLE] = "clear_enable",
 373};
 374
 375struct hisi_qm_hw_error {
 376        u32 int_msk;
 377        const char *msg;
 378};
 379
 380static const struct hisi_qm_hw_error qm_hw_error[] = {
 381        { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
 382        { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
 383        { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
 384        { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
 385        { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
 386        { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
 387        { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
 388        { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
 389        { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
 390        { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
 391        { .int_msk = BIT(10), .msg = "qm_db_timeout" },
 392        { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
 393        { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
 394        { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
 395        { .int_msk = BIT(14), .msg = "qm_flr_timeout" },
 396        { /* sentinel */ }
 397};
 398
 399static const char * const qm_db_timeout[] = {
 400        "sq", "cq", "eq", "aeq",
 401};
 402
 403static const char * const qm_fifo_overflow[] = {
 404        "cq", "eq", "aeq",
 405};
 406
 407static const char * const qm_s[] = {
 408        "init", "start", "close", "stop",
 409};
 410
 411static const char * const qp_s[] = {
 412        "none", "init", "start", "stop", "close",
 413};
 414
 415static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
 416{
 417        enum qm_state curr = atomic_read(&qm->status.flags);
 418        bool avail = false;
 419
 420        switch (curr) {
 421        case QM_INIT:
 422                if (new == QM_START || new == QM_CLOSE)
 423                        avail = true;
 424                break;
 425        case QM_START:
 426                if (new == QM_STOP)
 427                        avail = true;
 428                break;
 429        case QM_STOP:
 430                if (new == QM_CLOSE || new == QM_START)
 431                        avail = true;
 432                break;
 433        default:
 434                break;
 435        }
 436
 437        dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
 438                qm_s[curr], qm_s[new]);
 439
 440        if (!avail)
 441                dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
 442                         qm_s[curr], qm_s[new]);
 443
 444        return avail;
 445}
 446
 447static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
 448                              enum qp_state new)
 449{
 450        enum qm_state qm_curr = atomic_read(&qm->status.flags);
 451        enum qp_state qp_curr = 0;
 452        bool avail = false;
 453
 454        if (qp)
 455                qp_curr = atomic_read(&qp->qp_status.flags);
 456
 457        switch (new) {
 458        case QP_INIT:
 459                if (qm_curr == QM_START || qm_curr == QM_INIT)
 460                        avail = true;
 461                break;
 462        case QP_START:
 463                if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
 464                    (qm_curr == QM_START && qp_curr == QP_STOP))
 465                        avail = true;
 466                break;
 467        case QP_STOP:
 468                if ((qm_curr == QM_START && qp_curr == QP_START) ||
 469                    (qp_curr == QP_INIT))
 470                        avail = true;
 471                break;
 472        case QP_CLOSE:
 473                if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
 474                    (qm_curr == QM_START && qp_curr == QP_STOP) ||
 475                    (qm_curr == QM_STOP && qp_curr == QP_STOP)  ||
 476                    (qm_curr == QM_STOP && qp_curr == QP_INIT))
 477                        avail = true;
 478                break;
 479        default:
 480                break;
 481        }
 482
 483        dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
 484                qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
 485
 486        if (!avail)
 487                dev_warn(&qm->pdev->dev,
 488                         "Can not change qp state from %s to %s in QM %s\n",
 489                         qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
 490
 491        return avail;
 492}
 493
 494/* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
 495static int qm_wait_mb_ready(struct hisi_qm *qm)
 496{
 497        u32 val;
 498
 499        return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
 500                                          val, !((val >> QM_MB_BUSY_SHIFT) &
 501                                          0x1), POLL_PERIOD, POLL_TIMEOUT);
 502}
 503
 504/* 128 bit should be written to hardware at one time to trigger a mailbox */
 505static void qm_mb_write(struct hisi_qm *qm, const void *src)
 506{
 507        void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
 508        unsigned long tmp0 = 0, tmp1 = 0;
 509
 510        if (!IS_ENABLED(CONFIG_ARM64)) {
 511                memcpy_toio(fun_base, src, 16);
 512                wmb();
 513                return;
 514        }
 515
 516        asm volatile("ldp %0, %1, %3\n"
 517                     "stp %0, %1, %2\n"
 518                     "dsb sy\n"
 519                     : "=&r" (tmp0),
 520                       "=&r" (tmp1),
 521                       "+Q" (*((char __iomem *)fun_base))
 522                     : "Q" (*((char *)src))
 523                     : "memory");
 524}
 525
 526static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
 527                 bool op)
 528{
 529        struct qm_mailbox mailbox;
 530        int ret = 0;
 531
 532        dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
 533                queue, cmd, (unsigned long long)dma_addr);
 534
 535        mailbox.w0 = cpu_to_le16(cmd |
 536                     (op ? 0x1 << QM_MB_OP_SHIFT : 0) |
 537                     (0x1 << QM_MB_BUSY_SHIFT));
 538        mailbox.queue_num = cpu_to_le16(queue);
 539        mailbox.base_l = cpu_to_le32(lower_32_bits(dma_addr));
 540        mailbox.base_h = cpu_to_le32(upper_32_bits(dma_addr));
 541        mailbox.rsvd = 0;
 542
 543        mutex_lock(&qm->mailbox_lock);
 544
 545        if (unlikely(qm_wait_mb_ready(qm))) {
 546                ret = -EBUSY;
 547                dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
 548                goto busy_unlock;
 549        }
 550
 551        qm_mb_write(qm, &mailbox);
 552
 553        if (unlikely(qm_wait_mb_ready(qm))) {
 554                ret = -EBUSY;
 555                dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
 556                goto busy_unlock;
 557        }
 558
 559busy_unlock:
 560        mutex_unlock(&qm->mailbox_lock);
 561
 562        if (ret)
 563                atomic64_inc(&qm->debug.dfx.mb_err_cnt);
 564        return ret;
 565}
 566
 567static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 568{
 569        u64 doorbell;
 570
 571        doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
 572                   ((u64)index << QM_DB_INDEX_SHIFT_V1)  |
 573                   ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
 574
 575        writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
 576}
 577
 578static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 579{
 580        void __iomem *io_base = qm->io_base;
 581        u16 randata = 0;
 582        u64 doorbell;
 583
 584        if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
 585                io_base = qm->db_io_base + (u64)qn * qm->db_interval +
 586                          QM_DOORBELL_SQ_CQ_BASE_V2;
 587        else
 588                io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
 589
 590        doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
 591                   ((u64)randata << QM_DB_RAND_SHIFT_V2) |
 592                   ((u64)index << QM_DB_INDEX_SHIFT_V2)  |
 593                   ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
 594
 595        writeq(doorbell, io_base);
 596}
 597
 598static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 599{
 600        dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
 601                qn, cmd, index);
 602
 603        qm->ops->qm_db(qm, qn, cmd, index, priority);
 604}
 605
 606static int qm_dev_mem_reset(struct hisi_qm *qm)
 607{
 608        u32 val;
 609
 610        writel(0x1, qm->io_base + QM_MEM_START_INIT);
 611        return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
 612                                          val & BIT(0), POLL_PERIOD,
 613                                          POLL_TIMEOUT);
 614}
 615
 616static u32 qm_get_irq_num_v1(struct hisi_qm *qm)
 617{
 618        return QM_IRQ_NUM_V1;
 619}
 620
 621static u32 qm_get_irq_num_v2(struct hisi_qm *qm)
 622{
 623        if (qm->fun_type == QM_HW_PF)
 624                return QM_IRQ_NUM_PF_V2;
 625        else
 626                return QM_IRQ_NUM_VF_V2;
 627}
 628
 629static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe)
 630{
 631        u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
 632
 633        return &qm->qp_array[cqn];
 634}
 635
 636static void qm_cq_head_update(struct hisi_qp *qp)
 637{
 638        if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
 639                qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
 640                qp->qp_status.cq_head = 0;
 641        } else {
 642                qp->qp_status.cq_head++;
 643        }
 644}
 645
 646static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm)
 647{
 648        if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))
 649                return;
 650
 651        if (qp->event_cb) {
 652                qp->event_cb(qp);
 653                return;
 654        }
 655
 656        if (qp->req_cb) {
 657                struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
 658
 659                while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
 660                        dma_rmb();
 661                        qp->req_cb(qp, qp->sqe + qm->sqe_size *
 662                                   le16_to_cpu(cqe->sq_head));
 663                        qm_cq_head_update(qp);
 664                        cqe = qp->cqe + qp->qp_status.cq_head;
 665                        qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 666                              qp->qp_status.cq_head, 0);
 667                        atomic_dec(&qp->qp_status.used);
 668                }
 669
 670                /* set c_flag */
 671                qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 672                      qp->qp_status.cq_head, 1);
 673        }
 674}
 675
 676static void qm_work_process(struct work_struct *work)
 677{
 678        struct hisi_qm *qm = container_of(work, struct hisi_qm, work);
 679        struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
 680        struct hisi_qp *qp;
 681        int eqe_num = 0;
 682
 683        while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
 684                eqe_num++;
 685                qp = qm_to_hisi_qp(qm, eqe);
 686                qm_poll_qp(qp, qm);
 687
 688                if (qm->status.eq_head == QM_EQ_DEPTH - 1) {
 689                        qm->status.eqc_phase = !qm->status.eqc_phase;
 690                        eqe = qm->eqe;
 691                        qm->status.eq_head = 0;
 692                } else {
 693                        eqe++;
 694                        qm->status.eq_head++;
 695                }
 696
 697                if (eqe_num == QM_EQ_DEPTH / 2 - 1) {
 698                        eqe_num = 0;
 699                        qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 700                }
 701        }
 702
 703        qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 704}
 705
 706static irqreturn_t do_qm_irq(int irq, void *data)
 707{
 708        struct hisi_qm *qm = (struct hisi_qm *)data;
 709
 710        /* the workqueue created by device driver of QM */
 711        if (qm->wq)
 712                queue_work(qm->wq, &qm->work);
 713        else
 714                schedule_work(&qm->work);
 715
 716        return IRQ_HANDLED;
 717}
 718
 719static irqreturn_t qm_irq(int irq, void *data)
 720{
 721        struct hisi_qm *qm = data;
 722
 723        if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE))
 724                return do_qm_irq(irq, data);
 725
 726        atomic64_inc(&qm->debug.dfx.err_irq_cnt);
 727        dev_err(&qm->pdev->dev, "invalid int source\n");
 728        qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 729
 730        return IRQ_NONE;
 731}
 732
 733static irqreturn_t qm_aeq_irq(int irq, void *data)
 734{
 735        struct hisi_qm *qm = data;
 736        struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
 737        u32 type;
 738
 739        atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
 740        if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
 741                return IRQ_NONE;
 742
 743        while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
 744                type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
 745                if (type < ARRAY_SIZE(qm_fifo_overflow))
 746                        dev_err(&qm->pdev->dev, "%s overflow\n",
 747                                qm_fifo_overflow[type]);
 748                else
 749                        dev_err(&qm->pdev->dev, "unknown error type %u\n",
 750                                type);
 751
 752                if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
 753                        qm->status.aeqc_phase = !qm->status.aeqc_phase;
 754                        aeqe = qm->aeqe;
 755                        qm->status.aeq_head = 0;
 756                } else {
 757                        aeqe++;
 758                        qm->status.aeq_head++;
 759                }
 760
 761                qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
 762        }
 763
 764        return IRQ_HANDLED;
 765}
 766
 767static void qm_irq_unregister(struct hisi_qm *qm)
 768{
 769        struct pci_dev *pdev = qm->pdev;
 770
 771        free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
 772
 773        if (qm->ver == QM_HW_V1)
 774                return;
 775
 776        free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
 777
 778        if (qm->fun_type == QM_HW_PF)
 779                free_irq(pci_irq_vector(pdev,
 780                         QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
 781}
 782
 783static void qm_init_qp_status(struct hisi_qp *qp)
 784{
 785        struct hisi_qp_status *qp_status = &qp->qp_status;
 786
 787        qp_status->sq_tail = 0;
 788        qp_status->cq_head = 0;
 789        qp_status->cqc_phase = true;
 790        atomic_set(&qp_status->used, 0);
 791}
 792
 793static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
 794                            u32 number)
 795{
 796        u64 tmp = 0;
 797
 798        if (number > 0) {
 799                switch (type) {
 800                case SQC_VFT:
 801                        if (qm->ver == QM_HW_V1) {
 802                                tmp = QM_SQC_VFT_BUF_SIZE       |
 803                                      QM_SQC_VFT_SQC_SIZE       |
 804                                      QM_SQC_VFT_INDEX_NUMBER   |
 805                                      QM_SQC_VFT_VALID          |
 806                                      (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
 807                        } else {
 808                                tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
 809                                      QM_SQC_VFT_VALID |
 810                                      (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
 811                        }
 812                        break;
 813                case CQC_VFT:
 814                        if (qm->ver == QM_HW_V1) {
 815                                tmp = QM_CQC_VFT_BUF_SIZE       |
 816                                      QM_CQC_VFT_SQC_SIZE       |
 817                                      QM_CQC_VFT_INDEX_NUMBER   |
 818                                      QM_CQC_VFT_VALID;
 819                        } else {
 820                                tmp = QM_CQC_VFT_VALID;
 821                        }
 822                        break;
 823                }
 824        }
 825
 826        writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
 827        writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
 828}
 829
 830static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
 831                             u32 fun_num, u32 base, u32 number)
 832{
 833        unsigned int val;
 834        int ret;
 835
 836        ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 837                                         val & BIT(0), POLL_PERIOD,
 838                                         POLL_TIMEOUT);
 839        if (ret)
 840                return ret;
 841
 842        writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
 843        writel(type, qm->io_base + QM_VFT_CFG_TYPE);
 844        writel(fun_num, qm->io_base + QM_VFT_CFG);
 845
 846        qm_vft_data_cfg(qm, type, base, number);
 847
 848        writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
 849        writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
 850
 851        return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 852                                          val & BIT(0), POLL_PERIOD,
 853                                          POLL_TIMEOUT);
 854}
 855
 856/* The config should be conducted after qm_dev_mem_reset() */
 857static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
 858                              u32 number)
 859{
 860        int ret, i;
 861
 862        for (i = SQC_VFT; i <= CQC_VFT; i++) {
 863                ret = qm_set_vft_common(qm, i, fun_num, base, number);
 864                if (ret)
 865                        return ret;
 866        }
 867
 868        return 0;
 869}
 870
 871static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
 872{
 873        u64 sqc_vft;
 874        int ret;
 875
 876        ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
 877        if (ret)
 878                return ret;
 879
 880        sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 881                  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
 882        *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
 883        *number = (QM_SQC_VFT_NUM_MASK_v2 &
 884                   (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
 885
 886        return 0;
 887}
 888
 889static int qm_get_vf_qp_num(struct hisi_qm *qm, u32 fun_num)
 890{
 891        u32 remain_q_num, vfq_num;
 892        u32 num_vfs = qm->vfs_num;
 893
 894        vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs;
 895        if (vfq_num >= qm->max_qp_num)
 896                return qm->max_qp_num;
 897
 898        remain_q_num = (qm->ctrl_qp_num - qm->qp_num) % num_vfs;
 899        if (vfq_num + remain_q_num <= qm->max_qp_num)
 900                return fun_num == num_vfs ? vfq_num + remain_q_num : vfq_num;
 901
 902        /*
 903         * if vfq_num + remain_q_num > max_qp_num, the last VFs,
 904         * each with one more queue.
 905         */
 906        return fun_num + remain_q_num > num_vfs ? vfq_num + 1 : vfq_num;
 907}
 908
 909static struct hisi_qm *file_to_qm(struct debugfs_file *file)
 910{
 911        struct qm_debug *debug = file->debug;
 912
 913        return container_of(debug, struct hisi_qm, debug);
 914}
 915
 916static u32 current_q_read(struct debugfs_file *file)
 917{
 918        struct hisi_qm *qm = file_to_qm(file);
 919
 920        return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT;
 921}
 922
 923static int current_q_write(struct debugfs_file *file, u32 val)
 924{
 925        struct hisi_qm *qm = file_to_qm(file);
 926        u32 tmp;
 927
 928        if (val >= qm->debug.curr_qm_qp_num)
 929                return -EINVAL;
 930
 931        tmp = val << QM_DFX_QN_SHIFT |
 932              (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK);
 933        writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 934
 935        tmp = val << QM_DFX_QN_SHIFT |
 936              (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK);
 937        writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 938
 939        return 0;
 940}
 941
 942static u32 clear_enable_read(struct debugfs_file *file)
 943{
 944        struct hisi_qm *qm = file_to_qm(file);
 945
 946        return readl(qm->io_base + QM_DFX_CNT_CLR_CE);
 947}
 948
 949/* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */
 950static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl)
 951{
 952        struct hisi_qm *qm = file_to_qm(file);
 953
 954        if (rd_clr_ctrl > 1)
 955                return -EINVAL;
 956
 957        writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE);
 958
 959        return 0;
 960}
 961
 962static u32 current_qm_read(struct debugfs_file *file)
 963{
 964        struct hisi_qm *qm = file_to_qm(file);
 965
 966        return readl(qm->io_base + QM_DFX_MB_CNT_VF);
 967}
 968
 969static int current_qm_write(struct debugfs_file *file, u32 val)
 970{
 971        struct hisi_qm *qm = file_to_qm(file);
 972        u32 tmp;
 973
 974        if (val > qm->vfs_num)
 975                return -EINVAL;
 976
 977        /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
 978        if (!val)
 979                qm->debug.curr_qm_qp_num = qm->qp_num;
 980        else
 981                qm->debug.curr_qm_qp_num = qm_get_vf_qp_num(qm, val);
 982
 983        writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
 984        writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
 985
 986        tmp = val |
 987              (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
 988        writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 989
 990        tmp = val |
 991              (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
 992        writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 993
 994        return 0;
 995}
 996
 997static ssize_t qm_debug_read(struct file *filp, char __user *buf,
 998                             size_t count, loff_t *pos)
 999{
1000        struct debugfs_file *file = filp->private_data;
1001        enum qm_debug_file index = file->index;
1002        char tbuf[QM_DBG_TMP_BUF_LEN];
1003        u32 val;
1004        int ret;
1005
1006        mutex_lock(&file->lock);
1007        switch (index) {
1008        case CURRENT_QM:
1009                val = current_qm_read(file);
1010                break;
1011        case CURRENT_Q:
1012                val = current_q_read(file);
1013                break;
1014        case CLEAR_ENABLE:
1015                val = clear_enable_read(file);
1016                break;
1017        default:
1018                mutex_unlock(&file->lock);
1019                return -EINVAL;
1020        }
1021        mutex_unlock(&file->lock);
1022
1023        ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val);
1024        return simple_read_from_buffer(buf, count, pos, tbuf, ret);
1025}
1026
1027static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
1028                              size_t count, loff_t *pos)
1029{
1030        struct debugfs_file *file = filp->private_data;
1031        enum qm_debug_file index = file->index;
1032        unsigned long val;
1033        char tbuf[QM_DBG_TMP_BUF_LEN];
1034        int len, ret;
1035
1036        if (*pos != 0)
1037                return 0;
1038
1039        if (count >= QM_DBG_TMP_BUF_LEN)
1040                return -ENOSPC;
1041
1042        len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf,
1043                                     count);
1044        if (len < 0)
1045                return len;
1046
1047        tbuf[len] = '\0';
1048        if (kstrtoul(tbuf, 0, &val))
1049                return -EFAULT;
1050
1051        mutex_lock(&file->lock);
1052        switch (index) {
1053        case CURRENT_QM:
1054                ret = current_qm_write(file, val);
1055                break;
1056        case CURRENT_Q:
1057                ret = current_q_write(file, val);
1058                break;
1059        case CLEAR_ENABLE:
1060                ret = clear_enable_write(file, val);
1061                break;
1062        default:
1063                ret = -EINVAL;
1064        }
1065        mutex_unlock(&file->lock);
1066
1067        if (ret)
1068                return ret;
1069
1070        return count;
1071}
1072
1073static const struct file_operations qm_debug_fops = {
1074        .owner = THIS_MODULE,
1075        .open = simple_open,
1076        .read = qm_debug_read,
1077        .write = qm_debug_write,
1078};
1079
1080struct qm_dfx_registers {
1081        char  *reg_name;
1082        u64   reg_offset;
1083};
1084
1085#define CNT_CYC_REGS_NUM                10
1086static struct qm_dfx_registers qm_dfx_regs[] = {
1087        /* XXX_CNT are reading clear register */
1088        {"QM_ECC_1BIT_CNT               ",  0x104000ull},
1089        {"QM_ECC_MBIT_CNT               ",  0x104008ull},
1090        {"QM_DFX_MB_CNT                 ",  0x104018ull},
1091        {"QM_DFX_DB_CNT                 ",  0x104028ull},
1092        {"QM_DFX_SQE_CNT                ",  0x104038ull},
1093        {"QM_DFX_CQE_CNT                ",  0x104048ull},
1094        {"QM_DFX_SEND_SQE_TO_ACC_CNT    ",  0x104050ull},
1095        {"QM_DFX_WB_SQE_FROM_ACC_CNT    ",  0x104058ull},
1096        {"QM_DFX_ACC_FINISH_CNT         ",  0x104060ull},
1097        {"QM_DFX_CQE_ERR_CNT            ",  0x1040b4ull},
1098        {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
1099        {"QM_ECC_1BIT_INF               ",  0x104004ull},
1100        {"QM_ECC_MBIT_INF               ",  0x10400cull},
1101        {"QM_DFX_ACC_RDY_VLD0           ",  0x1040a0ull},
1102        {"QM_DFX_ACC_RDY_VLD1           ",  0x1040a4ull},
1103        {"QM_DFX_AXI_RDY_VLD            ",  0x1040a8ull},
1104        {"QM_DFX_FF_ST0                 ",  0x1040c8ull},
1105        {"QM_DFX_FF_ST1                 ",  0x1040ccull},
1106        {"QM_DFX_FF_ST2                 ",  0x1040d0ull},
1107        {"QM_DFX_FF_ST3                 ",  0x1040d4ull},
1108        {"QM_DFX_FF_ST4                 ",  0x1040d8ull},
1109        {"QM_DFX_FF_ST5                 ",  0x1040dcull},
1110        {"QM_DFX_FF_ST6                 ",  0x1040e0ull},
1111        {"QM_IN_IDLE_ST                 ",  0x1040e4ull},
1112        { NULL, 0}
1113};
1114
1115static struct qm_dfx_registers qm_vf_dfx_regs[] = {
1116        {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
1117        { NULL, 0}
1118};
1119
1120static int qm_regs_show(struct seq_file *s, void *unused)
1121{
1122        struct hisi_qm *qm = s->private;
1123        struct qm_dfx_registers *regs;
1124        u32 val;
1125
1126        if (qm->fun_type == QM_HW_PF)
1127                regs = qm_dfx_regs;
1128        else
1129                regs = qm_vf_dfx_regs;
1130
1131        while (regs->reg_name) {
1132                val = readl(qm->io_base + regs->reg_offset);
1133                seq_printf(s, "%s= 0x%08x\n", regs->reg_name, val);
1134                regs++;
1135        }
1136
1137        return 0;
1138}
1139
1140DEFINE_SHOW_ATTRIBUTE(qm_regs);
1141
1142static ssize_t qm_cmd_read(struct file *filp, char __user *buffer,
1143                           size_t count, loff_t *pos)
1144{
1145        char buf[QM_DBG_READ_LEN];
1146        int len;
1147
1148        len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n",
1149                        "Please echo help to cmd to get help information");
1150
1151        return simple_read_from_buffer(buffer, count, pos, buf, len);
1152}
1153
1154static void *qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size,
1155                          dma_addr_t *dma_addr)
1156{
1157        struct device *dev = &qm->pdev->dev;
1158        void *ctx_addr;
1159
1160        ctx_addr = kzalloc(ctx_size, GFP_KERNEL);
1161        if (!ctx_addr)
1162                return ERR_PTR(-ENOMEM);
1163
1164        *dma_addr = dma_map_single(dev, ctx_addr, ctx_size, DMA_FROM_DEVICE);
1165        if (dma_mapping_error(dev, *dma_addr)) {
1166                dev_err(dev, "DMA mapping error!\n");
1167                kfree(ctx_addr);
1168                return ERR_PTR(-ENOMEM);
1169        }
1170
1171        return ctx_addr;
1172}
1173
1174static void qm_ctx_free(struct hisi_qm *qm, size_t ctx_size,
1175                        const void *ctx_addr, dma_addr_t *dma_addr)
1176{
1177        struct device *dev = &qm->pdev->dev;
1178
1179        dma_unmap_single(dev, *dma_addr, ctx_size, DMA_FROM_DEVICE);
1180        kfree(ctx_addr);
1181}
1182
1183static int dump_show(struct hisi_qm *qm, void *info,
1184                     unsigned int info_size, char *info_name)
1185{
1186        struct device *dev = &qm->pdev->dev;
1187        u8 *info_buf, *info_curr = info;
1188        u32 i;
1189#define BYTE_PER_DW     4
1190
1191        info_buf = kzalloc(info_size, GFP_KERNEL);
1192        if (!info_buf)
1193                return -ENOMEM;
1194
1195        for (i = 0; i < info_size; i++, info_curr++) {
1196                if (i % BYTE_PER_DW == 0)
1197                        info_buf[i + 3UL] = *info_curr;
1198                else if (i % BYTE_PER_DW == 1)
1199                        info_buf[i + 1UL] = *info_curr;
1200                else if (i % BYTE_PER_DW == 2)
1201                        info_buf[i - 1] = *info_curr;
1202                else if (i % BYTE_PER_DW == 3)
1203                        info_buf[i - 3] = *info_curr;
1204        }
1205
1206        dev_info(dev, "%s DUMP\n", info_name);
1207        for (i = 0; i < info_size; i += BYTE_PER_DW) {
1208                pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW,
1209                        info_buf[i], info_buf[i + 1UL],
1210                        info_buf[i + 2UL], info_buf[i + 3UL]);
1211        }
1212
1213        kfree(info_buf);
1214
1215        return 0;
1216}
1217
1218static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1219{
1220        return qm_mb(qm, QM_MB_CMD_SQC, dma_addr, qp_id, 1);
1221}
1222
1223static int qm_dump_cqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1224{
1225        return qm_mb(qm, QM_MB_CMD_CQC, dma_addr, qp_id, 1);
1226}
1227
1228static int qm_sqc_dump(struct hisi_qm *qm, const char *s)
1229{
1230        struct device *dev = &qm->pdev->dev;
1231        struct qm_sqc *sqc, *sqc_curr;
1232        dma_addr_t sqc_dma;
1233        u32 qp_id;
1234        int ret;
1235
1236        if (!s)
1237                return -EINVAL;
1238
1239        ret = kstrtou32(s, 0, &qp_id);
1240        if (ret || qp_id >= qm->qp_num) {
1241                dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1);
1242                return -EINVAL;
1243        }
1244
1245        sqc = qm_ctx_alloc(qm, sizeof(*sqc), &sqc_dma);
1246        if (IS_ERR(sqc))
1247                return PTR_ERR(sqc);
1248
1249        ret = qm_dump_sqc_raw(qm, sqc_dma, qp_id);
1250        if (ret) {
1251                down_read(&qm->qps_lock);
1252                if (qm->sqc) {
1253                        sqc_curr = qm->sqc + qp_id;
1254
1255                        ret = dump_show(qm, sqc_curr, sizeof(*sqc),
1256                                        "SOFT SQC");
1257                        if (ret)
1258                                dev_info(dev, "Show soft sqc failed!\n");
1259                }
1260                up_read(&qm->qps_lock);
1261
1262                goto err_free_ctx;
1263        }
1264
1265        ret = dump_show(qm, sqc, sizeof(*sqc), "SQC");
1266        if (ret)
1267                dev_info(dev, "Show hw sqc failed!\n");
1268
1269err_free_ctx:
1270        qm_ctx_free(qm, sizeof(*sqc), sqc, &sqc_dma);
1271        return ret;
1272}
1273
1274static int qm_cqc_dump(struct hisi_qm *qm, const char *s)
1275{
1276        struct device *dev = &qm->pdev->dev;
1277        struct qm_cqc *cqc, *cqc_curr;
1278        dma_addr_t cqc_dma;
1279        u32 qp_id;
1280        int ret;
1281
1282        if (!s)
1283                return -EINVAL;
1284
1285        ret = kstrtou32(s, 0, &qp_id);
1286        if (ret || qp_id >= qm->qp_num) {
1287                dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1);
1288                return -EINVAL;
1289        }
1290
1291        cqc = qm_ctx_alloc(qm, sizeof(*cqc), &cqc_dma);
1292        if (IS_ERR(cqc))
1293                return PTR_ERR(cqc);
1294
1295        ret = qm_dump_cqc_raw(qm, cqc_dma, qp_id);
1296        if (ret) {
1297                down_read(&qm->qps_lock);
1298                if (qm->cqc) {
1299                        cqc_curr = qm->cqc + qp_id;
1300
1301                        ret = dump_show(qm, cqc_curr, sizeof(*cqc),
1302                                        "SOFT CQC");
1303                        if (ret)
1304                                dev_info(dev, "Show soft cqc failed!\n");
1305                }
1306                up_read(&qm->qps_lock);
1307
1308                goto err_free_ctx;
1309        }
1310
1311        ret = dump_show(qm, cqc, sizeof(*cqc), "CQC");
1312        if (ret)
1313                dev_info(dev, "Show hw cqc failed!\n");
1314
1315err_free_ctx:
1316        qm_ctx_free(qm, sizeof(*cqc), cqc, &cqc_dma);
1317        return ret;
1318}
1319
1320static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size,
1321                            int cmd, char *name)
1322{
1323        struct device *dev = &qm->pdev->dev;
1324        dma_addr_t xeqc_dma;
1325        void *xeqc;
1326        int ret;
1327
1328        if (strsep(&s, " ")) {
1329                dev_err(dev, "Please do not input extra characters!\n");
1330                return -EINVAL;
1331        }
1332
1333        xeqc = qm_ctx_alloc(qm, size, &xeqc_dma);
1334        if (IS_ERR(xeqc))
1335                return PTR_ERR(xeqc);
1336
1337        ret = qm_mb(qm, cmd, xeqc_dma, 0, 1);
1338        if (ret)
1339                goto err_free_ctx;
1340
1341        ret = dump_show(qm, xeqc, size, name);
1342        if (ret)
1343                dev_info(dev, "Show hw %s failed!\n", name);
1344
1345err_free_ctx:
1346        qm_ctx_free(qm, size, xeqc, &xeqc_dma);
1347        return ret;
1348}
1349
1350static int q_dump_param_parse(struct hisi_qm *qm, char *s,
1351                              u32 *e_id, u32 *q_id)
1352{
1353        struct device *dev = &qm->pdev->dev;
1354        unsigned int qp_num = qm->qp_num;
1355        char *presult;
1356        int ret;
1357
1358        presult = strsep(&s, " ");
1359        if (!presult) {
1360                dev_err(dev, "Please input qp number!\n");
1361                return -EINVAL;
1362        }
1363
1364        ret = kstrtou32(presult, 0, q_id);
1365        if (ret || *q_id >= qp_num) {
1366                dev_err(dev, "Please input qp num (0-%u)", qp_num - 1);
1367                return -EINVAL;
1368        }
1369
1370        presult = strsep(&s, " ");
1371        if (!presult) {
1372                dev_err(dev, "Please input sqe number!\n");
1373                return -EINVAL;
1374        }
1375
1376        ret = kstrtou32(presult, 0, e_id);
1377        if (ret || *e_id >= QM_Q_DEPTH) {
1378                dev_err(dev, "Please input sqe num (0-%d)", QM_Q_DEPTH - 1);
1379                return -EINVAL;
1380        }
1381
1382        if (strsep(&s, " ")) {
1383                dev_err(dev, "Please do not input extra characters!\n");
1384                return -EINVAL;
1385        }
1386
1387        return 0;
1388}
1389
1390static int qm_sq_dump(struct hisi_qm *qm, char *s)
1391{
1392        struct device *dev = &qm->pdev->dev;
1393        void *sqe, *sqe_curr;
1394        struct hisi_qp *qp;
1395        u32 qp_id, sqe_id;
1396        int ret;
1397
1398        ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id);
1399        if (ret)
1400                return ret;
1401
1402        sqe = kzalloc(qm->sqe_size * QM_Q_DEPTH, GFP_KERNEL);
1403        if (!sqe)
1404                return -ENOMEM;
1405
1406        qp = &qm->qp_array[qp_id];
1407        memcpy(sqe, qp->sqe, qm->sqe_size * QM_Q_DEPTH);
1408        sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size);
1409        memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK,
1410               qm->debug.sqe_mask_len);
1411
1412        ret = dump_show(qm, sqe_curr, qm->sqe_size, "SQE");
1413        if (ret)
1414                dev_info(dev, "Show sqe failed!\n");
1415
1416        kfree(sqe);
1417
1418        return ret;
1419}
1420
1421static int qm_cq_dump(struct hisi_qm *qm, char *s)
1422{
1423        struct device *dev = &qm->pdev->dev;
1424        struct qm_cqe *cqe_curr;
1425        struct hisi_qp *qp;
1426        u32 qp_id, cqe_id;
1427        int ret;
1428
1429        ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id);
1430        if (ret)
1431                return ret;
1432
1433        qp = &qm->qp_array[qp_id];
1434        cqe_curr = qp->cqe + cqe_id;
1435        ret = dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE");
1436        if (ret)
1437                dev_info(dev, "Show cqe failed!\n");
1438
1439        return ret;
1440}
1441
1442static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s,
1443                          size_t size, char *name)
1444{
1445        struct device *dev = &qm->pdev->dev;
1446        void *xeqe;
1447        u32 xeqe_id;
1448        int ret;
1449
1450        if (!s)
1451                return -EINVAL;
1452
1453        ret = kstrtou32(s, 0, &xeqe_id);
1454        if (ret)
1455                return -EINVAL;
1456
1457        if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) {
1458                dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1);
1459                return -EINVAL;
1460        } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) {
1461                dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1);
1462                return -EINVAL;
1463        }
1464
1465        down_read(&qm->qps_lock);
1466
1467        if (qm->eqe && !strcmp(name, "EQE")) {
1468                xeqe = qm->eqe + xeqe_id;
1469        } else if (qm->aeqe && !strcmp(name, "AEQE")) {
1470                xeqe = qm->aeqe + xeqe_id;
1471        } else {
1472                ret = -EINVAL;
1473                goto err_unlock;
1474        }
1475
1476        ret = dump_show(qm, xeqe, size, name);
1477        if (ret)
1478                dev_info(dev, "Show %s failed!\n", name);
1479
1480err_unlock:
1481        up_read(&qm->qps_lock);
1482        return ret;
1483}
1484
1485static int qm_dbg_help(struct hisi_qm *qm, char *s)
1486{
1487        struct device *dev = &qm->pdev->dev;
1488
1489        if (strsep(&s, " ")) {
1490                dev_err(dev, "Please do not input extra characters!\n");
1491                return -EINVAL;
1492        }
1493
1494        dev_info(dev, "available commands:\n");
1495        dev_info(dev, "sqc <num>\n");
1496        dev_info(dev, "cqc <num>\n");
1497        dev_info(dev, "eqc\n");
1498        dev_info(dev, "aeqc\n");
1499        dev_info(dev, "sq <num> <e>\n");
1500        dev_info(dev, "cq <num> <e>\n");
1501        dev_info(dev, "eq <e>\n");
1502        dev_info(dev, "aeq <e>\n");
1503
1504        return 0;
1505}
1506
1507static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf)
1508{
1509        struct device *dev = &qm->pdev->dev;
1510        char *presult, *s, *s_tmp;
1511        int ret;
1512
1513        s = kstrdup(cmd_buf, GFP_KERNEL);
1514        if (!s)
1515                return -ENOMEM;
1516
1517        s_tmp = s;
1518        presult = strsep(&s, " ");
1519        if (!presult) {
1520                ret = -EINVAL;
1521                goto err_buffer_free;
1522        }
1523
1524        if (!strcmp(presult, "sqc"))
1525                ret = qm_sqc_dump(qm, s);
1526        else if (!strcmp(presult, "cqc"))
1527                ret = qm_cqc_dump(qm, s);
1528        else if (!strcmp(presult, "eqc"))
1529                ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_eqc),
1530                                       QM_MB_CMD_EQC, "EQC");
1531        else if (!strcmp(presult, "aeqc"))
1532                ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_aeqc),
1533                                       QM_MB_CMD_AEQC, "AEQC");
1534        else if (!strcmp(presult, "sq"))
1535                ret = qm_sq_dump(qm, s);
1536        else if (!strcmp(presult, "cq"))
1537                ret = qm_cq_dump(qm, s);
1538        else if (!strcmp(presult, "eq"))
1539                ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_eqe), "EQE");
1540        else if (!strcmp(presult, "aeq"))
1541                ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_aeqe), "AEQE");
1542        else if (!strcmp(presult, "help"))
1543                ret = qm_dbg_help(qm, s);
1544        else
1545                ret = -EINVAL;
1546
1547        if (ret)
1548                dev_info(dev, "Please echo help\n");
1549
1550err_buffer_free:
1551        kfree(s_tmp);
1552
1553        return ret;
1554}
1555
1556static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer,
1557                            size_t count, loff_t *pos)
1558{
1559        struct hisi_qm *qm = filp->private_data;
1560        char *cmd_buf, *cmd_buf_tmp;
1561        int ret;
1562
1563        if (*pos)
1564                return 0;
1565
1566        /* Judge if the instance is being reset. */
1567        if (unlikely(atomic_read(&qm->status.flags) == QM_STOP))
1568                return 0;
1569
1570        if (count > QM_DBG_WRITE_LEN)
1571                return -ENOSPC;
1572
1573        cmd_buf = kzalloc(count + 1, GFP_KERNEL);
1574        if (!cmd_buf)
1575                return -ENOMEM;
1576
1577        if (copy_from_user(cmd_buf, buffer, count)) {
1578                kfree(cmd_buf);
1579                return -EFAULT;
1580        }
1581
1582        cmd_buf[count] = '\0';
1583
1584        cmd_buf_tmp = strchr(cmd_buf, '\n');
1585        if (cmd_buf_tmp) {
1586                *cmd_buf_tmp = '\0';
1587                count = cmd_buf_tmp - cmd_buf + 1;
1588        }
1589
1590        ret = qm_cmd_write_dump(qm, cmd_buf);
1591        if (ret) {
1592                kfree(cmd_buf);
1593                return ret;
1594        }
1595
1596        kfree(cmd_buf);
1597
1598        return count;
1599}
1600
1601static const struct file_operations qm_cmd_fops = {
1602        .owner = THIS_MODULE,
1603        .open = simple_open,
1604        .read = qm_cmd_read,
1605        .write = qm_cmd_write,
1606};
1607
1608static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
1609                                   enum qm_debug_file index)
1610{
1611        struct debugfs_file *file = qm->debug.files + index;
1612
1613        debugfs_create_file(qm_debug_file_name[index], 0600, dir, file,
1614                            &qm_debug_fops);
1615
1616        file->index = index;
1617        mutex_init(&file->lock);
1618        file->debug = &qm->debug;
1619}
1620
1621static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1622{
1623        writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1624}
1625
1626static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1627{
1628        u32 irq_enable = ce | nfe | fe;
1629        u32 irq_unmask = ~irq_enable;
1630
1631        qm->error_mask = ce | nfe | fe;
1632
1633        /* clear QM hw residual error source */
1634        writel(QM_ABNORMAL_INT_SOURCE_CLR,
1635               qm->io_base + QM_ABNORMAL_INT_SOURCE);
1636
1637        /* configure error type */
1638        writel(ce, qm->io_base + QM_RAS_CE_ENABLE);
1639        writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1640        writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1641        writel(fe, qm->io_base + QM_RAS_FE_ENABLE);
1642
1643        irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1644        writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1645}
1646
1647static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1648{
1649        writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1650}
1651
1652static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1653{
1654        const struct hisi_qm_hw_error *err;
1655        struct device *dev = &qm->pdev->dev;
1656        u32 reg_val, type, vf_num;
1657        int i;
1658
1659        for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
1660                err = &qm_hw_error[i];
1661                if (!(err->int_msk & error_status))
1662                        continue;
1663
1664                dev_err(dev, "%s [error status=0x%x] found\n",
1665                        err->msg, err->int_msk);
1666
1667                if (err->int_msk & QM_DB_TIMEOUT) {
1668                        reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
1669                        type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1670                               QM_DB_TIMEOUT_TYPE_SHIFT;
1671                        vf_num = reg_val & QM_DB_TIMEOUT_VF;
1672                        dev_err(dev, "qm %s doorbell timeout in function %u\n",
1673                                qm_db_timeout[type], vf_num);
1674                } else if (err->int_msk & QM_OF_FIFO_OF) {
1675                        reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
1676                        type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1677                               QM_FIFO_OVERFLOW_TYPE_SHIFT;
1678                        vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1679
1680                        if (type < ARRAY_SIZE(qm_fifo_overflow))
1681                                dev_err(dev, "qm %s fifo overflow in function %u\n",
1682                                        qm_fifo_overflow[type], vf_num);
1683                        else
1684                                dev_err(dev, "unknown error type\n");
1685                }
1686        }
1687}
1688
1689static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
1690{
1691        u32 error_status, tmp, val;
1692
1693        /* read err sts */
1694        tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
1695        error_status = qm->error_mask & tmp;
1696
1697        if (error_status) {
1698                if (error_status & QM_ECC_MBIT)
1699                        qm->err_status.is_qm_ecc_mbit = true;
1700
1701                qm_log_hw_error(qm, error_status);
1702                val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE;
1703                /* ce error does not need to be reset */
1704                if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) {
1705                        writel(error_status, qm->io_base +
1706                               QM_ABNORMAL_INT_SOURCE);
1707                        writel(qm->err_info.nfe,
1708                               qm->io_base + QM_RAS_NFE_ENABLE);
1709                        return ACC_ERR_RECOVERED;
1710                }
1711
1712                return ACC_ERR_NEED_RESET;
1713        }
1714
1715        return ACC_ERR_RECOVERED;
1716}
1717
1718static int qm_stop_qp(struct hisi_qp *qp)
1719{
1720        return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
1721}
1722
1723static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1724        .qm_db = qm_db_v1,
1725        .get_irq_num = qm_get_irq_num_v1,
1726        .hw_error_init = qm_hw_error_init_v1,
1727};
1728
1729static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1730        .get_vft = qm_get_vft_v2,
1731        .qm_db = qm_db_v2,
1732        .get_irq_num = qm_get_irq_num_v2,
1733        .hw_error_init = qm_hw_error_init_v2,
1734        .hw_error_uninit = qm_hw_error_uninit_v2,
1735        .hw_error_handle = qm_hw_error_handle_v2,
1736};
1737
1738static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
1739        .get_vft = qm_get_vft_v2,
1740        .qm_db = qm_db_v2,
1741        .get_irq_num = qm_get_irq_num_v2,
1742        .hw_error_init = qm_hw_error_init_v2,
1743        .hw_error_uninit = qm_hw_error_uninit_v2,
1744        .hw_error_handle = qm_hw_error_handle_v2,
1745        .stop_qp = qm_stop_qp,
1746};
1747
1748static void *qm_get_avail_sqe(struct hisi_qp *qp)
1749{
1750        struct hisi_qp_status *qp_status = &qp->qp_status;
1751        u16 sq_tail = qp_status->sq_tail;
1752
1753        if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1))
1754                return NULL;
1755
1756        return qp->sqe + sq_tail * qp->qm->sqe_size;
1757}
1758
1759static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
1760{
1761        struct device *dev = &qm->pdev->dev;
1762        struct hisi_qp *qp;
1763        int qp_id;
1764
1765        if (!qm_qp_avail_state(qm, NULL, QP_INIT))
1766                return ERR_PTR(-EPERM);
1767
1768        if (qm->qp_in_used == qm->qp_num) {
1769                dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1770                                     qm->qp_num);
1771                atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1772                return ERR_PTR(-EBUSY);
1773        }
1774
1775        qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
1776        if (qp_id < 0) {
1777                dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
1778                                    qm->qp_num);
1779                atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
1780                return ERR_PTR(-EBUSY);
1781        }
1782
1783        qp = &qm->qp_array[qp_id];
1784
1785        memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH);
1786
1787        qp->event_cb = NULL;
1788        qp->req_cb = NULL;
1789        qp->qp_id = qp_id;
1790        qp->alg_type = alg_type;
1791        qp->is_in_kernel = true;
1792        qm->qp_in_used++;
1793        atomic_set(&qp->qp_status.flags, QP_INIT);
1794
1795        return qp;
1796}
1797
1798/**
1799 * hisi_qm_create_qp() - Create a queue pair from qm.
1800 * @qm: The qm we create a qp from.
1801 * @alg_type: Accelerator specific algorithm type in sqc.
1802 *
1803 * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
1804 * qp memory fails.
1805 */
1806struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
1807{
1808        struct hisi_qp *qp;
1809
1810        down_write(&qm->qps_lock);
1811        qp = qm_create_qp_nolock(qm, alg_type);
1812        up_write(&qm->qps_lock);
1813
1814        return qp;
1815}
1816EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
1817
1818/**
1819 * hisi_qm_release_qp() - Release a qp back to its qm.
1820 * @qp: The qp we want to release.
1821 *
1822 * This function releases the resource of a qp.
1823 */
1824void hisi_qm_release_qp(struct hisi_qp *qp)
1825{
1826        struct hisi_qm *qm = qp->qm;
1827
1828        down_write(&qm->qps_lock);
1829
1830        if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
1831                up_write(&qm->qps_lock);
1832                return;
1833        }
1834
1835        qm->qp_in_used--;
1836        idr_remove(&qm->qp_idr, qp->qp_id);
1837
1838        up_write(&qm->qps_lock);
1839}
1840EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
1841
1842static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1843{
1844        struct hisi_qm *qm = qp->qm;
1845        struct device *dev = &qm->pdev->dev;
1846        enum qm_hw_ver ver = qm->ver;
1847        struct qm_sqc *sqc;
1848        dma_addr_t sqc_dma;
1849        int ret;
1850
1851        sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
1852        if (!sqc)
1853                return -ENOMEM;
1854
1855        INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
1856        if (ver == QM_HW_V1) {
1857                sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
1858                sqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
1859        } else {
1860                sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size));
1861                sqc->w8 = 0; /* rand_qc */
1862        }
1863        sqc->cq_num = cpu_to_le16(qp_id);
1864        sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
1865
1866        if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
1867                sqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<
1868                                       QM_QC_PASID_ENABLE_SHIFT);
1869
1870        sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
1871                                 DMA_TO_DEVICE);
1872        if (dma_mapping_error(dev, sqc_dma)) {
1873                kfree(sqc);
1874                return -ENOMEM;
1875        }
1876
1877        ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
1878        dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
1879        kfree(sqc);
1880
1881        return ret;
1882}
1883
1884static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1885{
1886        struct hisi_qm *qm = qp->qm;
1887        struct device *dev = &qm->pdev->dev;
1888        enum qm_hw_ver ver = qm->ver;
1889        struct qm_cqc *cqc;
1890        dma_addr_t cqc_dma;
1891        int ret;
1892
1893        cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
1894        if (!cqc)
1895                return -ENOMEM;
1896
1897        INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
1898        if (ver == QM_HW_V1) {
1899                cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0,
1900                                                        QM_QC_CQE_SIZE));
1901                cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
1902        } else {
1903                cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE));
1904                cqc->w8 = 0; /* rand_qc */
1905        }
1906        cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
1907
1908        if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
1909                cqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE);
1910
1911        cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
1912                                 DMA_TO_DEVICE);
1913        if (dma_mapping_error(dev, cqc_dma)) {
1914                kfree(cqc);
1915                return -ENOMEM;
1916        }
1917
1918        ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
1919        dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
1920        kfree(cqc);
1921
1922        return ret;
1923}
1924
1925static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
1926{
1927        int ret;
1928
1929        qm_init_qp_status(qp);
1930
1931        ret = qm_sq_ctx_cfg(qp, qp_id, pasid);
1932        if (ret)
1933                return ret;
1934
1935        return qm_cq_ctx_cfg(qp, qp_id, pasid);
1936}
1937
1938static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
1939{
1940        struct hisi_qm *qm = qp->qm;
1941        struct device *dev = &qm->pdev->dev;
1942        int qp_id = qp->qp_id;
1943        u32 pasid = arg;
1944        int ret;
1945
1946        if (!qm_qp_avail_state(qm, qp, QP_START))
1947                return -EPERM;
1948
1949        ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
1950        if (ret)
1951                return ret;
1952
1953        atomic_set(&qp->qp_status.flags, QP_START);
1954        dev_dbg(dev, "queue %d started\n", qp_id);
1955
1956        return 0;
1957}
1958
1959/**
1960 * hisi_qm_start_qp() - Start a qp into running.
1961 * @qp: The qp we want to start to run.
1962 * @arg: Accelerator specific argument.
1963 *
1964 * After this function, qp can receive request from user. Return 0 if
1965 * successful, Return -EBUSY if failed.
1966 */
1967int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
1968{
1969        struct hisi_qm *qm = qp->qm;
1970        int ret;
1971
1972        down_write(&qm->qps_lock);
1973        ret = qm_start_qp_nolock(qp, arg);
1974        up_write(&qm->qps_lock);
1975
1976        return ret;
1977}
1978EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
1979
1980/**
1981 * qp_stop_fail_cb() - call request cb.
1982 * @qp: stopped failed qp.
1983 *
1984 * Callback function should be called whether task completed or not.
1985 */
1986static void qp_stop_fail_cb(struct hisi_qp *qp)
1987{
1988        int qp_used = atomic_read(&qp->qp_status.used);
1989        u16 cur_tail = qp->qp_status.sq_tail;
1990        u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH;
1991        struct hisi_qm *qm = qp->qm;
1992        u16 pos;
1993        int i;
1994
1995        for (i = 0; i < qp_used; i++) {
1996                pos = (i + cur_head) % QM_Q_DEPTH;
1997                qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
1998                atomic_dec(&qp->qp_status.used);
1999        }
2000}
2001
2002/**
2003 * qm_drain_qp() - Drain a qp.
2004 * @qp: The qp we want to drain.
2005 *
2006 * Determine whether the queue is cleared by judging the tail pointers of
2007 * sq and cq.
2008 */
2009static int qm_drain_qp(struct hisi_qp *qp)
2010{
2011        size_t size = sizeof(struct qm_sqc) + sizeof(struct qm_cqc);
2012        struct hisi_qm *qm = qp->qm;
2013        struct device *dev = &qm->pdev->dev;
2014        struct qm_sqc *sqc;
2015        struct qm_cqc *cqc;
2016        dma_addr_t dma_addr;
2017        int ret = 0, i = 0;
2018        void *addr;
2019
2020        /*
2021         * No need to judge if ECC multi-bit error occurs because the
2022         * master OOO will be blocked.
2023         */
2024        if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit)
2025                return 0;
2026
2027        /* Kunpeng930 supports drain qp by device */
2028        if (qm->ops->stop_qp) {
2029                ret = qm->ops->stop_qp(qp);
2030                if (ret)
2031                        dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id);
2032                return ret;
2033        }
2034
2035        addr = qm_ctx_alloc(qm, size, &dma_addr);
2036        if (IS_ERR(addr)) {
2037                dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n");
2038                return -ENOMEM;
2039        }
2040
2041        while (++i) {
2042                ret = qm_dump_sqc_raw(qm, dma_addr, qp->qp_id);
2043                if (ret) {
2044                        dev_err_ratelimited(dev, "Failed to dump sqc!\n");
2045                        break;
2046                }
2047                sqc = addr;
2048
2049                ret = qm_dump_cqc_raw(qm, (dma_addr + sizeof(struct qm_sqc)),
2050                                      qp->qp_id);
2051                if (ret) {
2052                        dev_err_ratelimited(dev, "Failed to dump cqc!\n");
2053                        break;
2054                }
2055                cqc = addr + sizeof(struct qm_sqc);
2056
2057                if ((sqc->tail == cqc->tail) &&
2058                    (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
2059                        break;
2060
2061                if (i == MAX_WAIT_COUNTS) {
2062                        dev_err(dev, "Fail to empty queue %u!\n", qp->qp_id);
2063                        ret = -EBUSY;
2064                        break;
2065                }
2066
2067                usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
2068        }
2069
2070        qm_ctx_free(qm, size, addr, &dma_addr);
2071
2072        return ret;
2073}
2074
2075static int qm_stop_qp_nolock(struct hisi_qp *qp)
2076{
2077        struct device *dev = &qp->qm->pdev->dev;
2078        int ret;
2079
2080        /*
2081         * It is allowed to stop and release qp when reset, If the qp is
2082         * stopped when reset but still want to be released then, the
2083         * is_resetting flag should be set negative so that this qp will not
2084         * be restarted after reset.
2085         */
2086        if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
2087                qp->is_resetting = false;
2088                return 0;
2089        }
2090
2091        if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
2092                return -EPERM;
2093
2094        atomic_set(&qp->qp_status.flags, QP_STOP);
2095
2096        ret = qm_drain_qp(qp);
2097        if (ret)
2098                dev_err(dev, "Failed to drain out data for stopping!\n");
2099
2100        if (qp->qm->wq)
2101                flush_workqueue(qp->qm->wq);
2102        else
2103                flush_work(&qp->qm->work);
2104
2105        if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
2106                qp_stop_fail_cb(qp);
2107
2108        dev_dbg(dev, "stop queue %u!", qp->qp_id);
2109
2110        return 0;
2111}
2112
2113/**
2114 * hisi_qm_stop_qp() - Stop a qp in qm.
2115 * @qp: The qp we want to stop.
2116 *
2117 * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
2118 */
2119int hisi_qm_stop_qp(struct hisi_qp *qp)
2120{
2121        int ret;
2122
2123        down_write(&qp->qm->qps_lock);
2124        ret = qm_stop_qp_nolock(qp);
2125        up_write(&qp->qm->qps_lock);
2126
2127        return ret;
2128}
2129EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
2130
2131/**
2132 * hisi_qp_send() - Queue up a task in the hardware queue.
2133 * @qp: The qp in which to put the message.
2134 * @msg: The message.
2135 *
2136 * This function will return -EBUSY if qp is currently full, and -EAGAIN
2137 * if qp related qm is resetting.
2138 *
2139 * Note: This function may run with qm_irq_thread and ACC reset at same time.
2140 *       It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
2141 *       reset may happen, we have no lock here considering performance. This
2142 *       causes current qm_db sending fail or can not receive sended sqe. QM
2143 *       sync/async receive function should handle the error sqe. ACC reset
2144 *       done function should clear used sqe to 0.
2145 */
2146int hisi_qp_send(struct hisi_qp *qp, const void *msg)
2147{
2148        struct hisi_qp_status *qp_status = &qp->qp_status;
2149        u16 sq_tail = qp_status->sq_tail;
2150        u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
2151        void *sqe = qm_get_avail_sqe(qp);
2152
2153        if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
2154                     atomic_read(&qp->qm->status.flags) == QM_STOP ||
2155                     qp->is_resetting)) {
2156                dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
2157                return -EAGAIN;
2158        }
2159
2160        if (!sqe)
2161                return -EBUSY;
2162
2163        memcpy(sqe, msg, qp->qm->sqe_size);
2164
2165        qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
2166        atomic_inc(&qp->qp_status.used);
2167        qp_status->sq_tail = sq_tail_next;
2168
2169        return 0;
2170}
2171EXPORT_SYMBOL_GPL(hisi_qp_send);
2172
2173static void hisi_qm_cache_wb(struct hisi_qm *qm)
2174{
2175        unsigned int val;
2176
2177        if (qm->ver == QM_HW_V1)
2178                return;
2179
2180        writel(0x1, qm->io_base + QM_CACHE_WB_START);
2181        if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2182                                       val, val & BIT(0), POLL_PERIOD,
2183                                       POLL_TIMEOUT))
2184                dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2185}
2186
2187static void qm_qp_event_notifier(struct hisi_qp *qp)
2188{
2189        wake_up_interruptible(&qp->uacce_q->wait);
2190}
2191
2192static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2193{
2194        return hisi_qm_get_free_qp_num(uacce->priv);
2195}
2196
2197static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2198                                   unsigned long arg,
2199                                   struct uacce_queue *q)
2200{
2201        struct hisi_qm *qm = uacce->priv;
2202        struct hisi_qp *qp;
2203        u8 alg_type = 0;
2204
2205        qp = hisi_qm_create_qp(qm, alg_type);
2206        if (IS_ERR(qp))
2207                return PTR_ERR(qp);
2208
2209        q->priv = qp;
2210        q->uacce = uacce;
2211        qp->uacce_q = q;
2212        qp->event_cb = qm_qp_event_notifier;
2213        qp->pasid = arg;
2214        qp->is_in_kernel = false;
2215
2216        return 0;
2217}
2218
2219static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2220{
2221        struct hisi_qp *qp = q->priv;
2222
2223        hisi_qm_cache_wb(qp->qm);
2224        hisi_qm_release_qp(qp);
2225}
2226
2227/* map sq/cq/doorbell to user space */
2228static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2229                              struct vm_area_struct *vma,
2230                              struct uacce_qfile_region *qfr)
2231{
2232        struct hisi_qp *qp = q->priv;
2233        struct hisi_qm *qm = qp->qm;
2234        resource_size_t phys_base = qm->db_phys_base +
2235                                    qp->qp_id * qm->db_interval;
2236        size_t sz = vma->vm_end - vma->vm_start;
2237        struct pci_dev *pdev = qm->pdev;
2238        struct device *dev = &pdev->dev;
2239        unsigned long vm_pgoff;
2240        int ret;
2241
2242        switch (qfr->type) {
2243        case UACCE_QFRT_MMIO:
2244                if (qm->ver == QM_HW_V1) {
2245                        if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2246                                return -EINVAL;
2247                } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) {
2248                        if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2249                            QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2250                                return -EINVAL;
2251                } else {
2252                        if (sz > qm->db_interval)
2253                                return -EINVAL;
2254                }
2255
2256                vma->vm_flags |= VM_IO;
2257
2258                return remap_pfn_range(vma, vma->vm_start,
2259                                       phys_base >> PAGE_SHIFT,
2260                                       sz, pgprot_noncached(vma->vm_page_prot));
2261        case UACCE_QFRT_DUS:
2262                if (sz != qp->qdma.size)
2263                        return -EINVAL;
2264
2265                /*
2266                 * dma_mmap_coherent() requires vm_pgoff as 0
2267                 * restore vm_pfoff to initial value for mmap()
2268                 */
2269                vm_pgoff = vma->vm_pgoff;
2270                vma->vm_pgoff = 0;
2271                ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2272                                        qp->qdma.dma, sz);
2273                vma->vm_pgoff = vm_pgoff;
2274                return ret;
2275
2276        default:
2277                return -EINVAL;
2278        }
2279}
2280
2281static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
2282{
2283        struct hisi_qp *qp = q->priv;
2284
2285        return hisi_qm_start_qp(qp, qp->pasid);
2286}
2287
2288static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
2289{
2290        hisi_qm_stop_qp(q->priv);
2291}
2292
2293static void qm_set_sqctype(struct uacce_queue *q, u16 type)
2294{
2295        struct hisi_qm *qm = q->uacce->priv;
2296        struct hisi_qp *qp = q->priv;
2297
2298        down_write(&qm->qps_lock);
2299        qp->alg_type = type;
2300        up_write(&qm->qps_lock);
2301}
2302
2303static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
2304                                unsigned long arg)
2305{
2306        struct hisi_qp *qp = q->priv;
2307        struct hisi_qp_ctx qp_ctx;
2308
2309        if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
2310                if (copy_from_user(&qp_ctx, (void __user *)arg,
2311                                   sizeof(struct hisi_qp_ctx)))
2312                        return -EFAULT;
2313
2314                if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1)
2315                        return -EINVAL;
2316
2317                qm_set_sqctype(q, qp_ctx.qc_type);
2318                qp_ctx.id = qp->qp_id;
2319
2320                if (copy_to_user((void __user *)arg, &qp_ctx,
2321                                 sizeof(struct hisi_qp_ctx)))
2322                        return -EFAULT;
2323        } else {
2324                return -EINVAL;
2325        }
2326
2327        return 0;
2328}
2329
2330static const struct uacce_ops uacce_qm_ops = {
2331        .get_available_instances = hisi_qm_get_available_instances,
2332        .get_queue = hisi_qm_uacce_get_queue,
2333        .put_queue = hisi_qm_uacce_put_queue,
2334        .start_queue = hisi_qm_uacce_start_queue,
2335        .stop_queue = hisi_qm_uacce_stop_queue,
2336        .mmap = hisi_qm_uacce_mmap,
2337        .ioctl = hisi_qm_uacce_ioctl,
2338};
2339
2340static int qm_alloc_uacce(struct hisi_qm *qm)
2341{
2342        struct pci_dev *pdev = qm->pdev;
2343        struct uacce_device *uacce;
2344        unsigned long mmio_page_nr;
2345        unsigned long dus_page_nr;
2346        struct uacce_interface interface = {
2347                .flags = UACCE_DEV_SVA,
2348                .ops = &uacce_qm_ops,
2349        };
2350        int ret;
2351
2352        ret = strscpy(interface.name, pdev->driver->name,
2353                      sizeof(interface.name));
2354        if (ret < 0)
2355                return -ENAMETOOLONG;
2356
2357        uacce = uacce_alloc(&pdev->dev, &interface);
2358        if (IS_ERR(uacce))
2359                return PTR_ERR(uacce);
2360
2361        if (uacce->flags & UACCE_DEV_SVA && qm->mode == UACCE_MODE_SVA) {
2362                qm->use_sva = true;
2363        } else {
2364                /* only consider sva case */
2365                uacce_remove(uacce);
2366                qm->uacce = NULL;
2367                return -EINVAL;
2368        }
2369
2370        uacce->is_vf = pdev->is_virtfn;
2371        uacce->priv = qm;
2372        uacce->algs = qm->algs;
2373
2374        if (qm->ver == QM_HW_V1)
2375                uacce->api_ver = HISI_QM_API_VER_BASE;
2376        else if (qm->ver == QM_HW_V2)
2377                uacce->api_ver = HISI_QM_API_VER2_BASE;
2378        else
2379                uacce->api_ver = HISI_QM_API_VER3_BASE;
2380
2381        if (qm->ver == QM_HW_V1)
2382                mmio_page_nr = QM_DOORBELL_PAGE_NR;
2383        else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation)
2384                mmio_page_nr = QM_DOORBELL_PAGE_NR +
2385                        QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
2386        else
2387                mmio_page_nr = qm->db_interval / PAGE_SIZE;
2388
2389        dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH +
2390                       sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT;
2391
2392        uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
2393        uacce->qf_pg_num[UACCE_QFRT_DUS]  = dus_page_nr;
2394
2395        qm->uacce = uacce;
2396
2397        return 0;
2398}
2399
2400/**
2401 * qm_frozen() - Try to froze QM to cut continuous queue request. If
2402 * there is user on the QM, return failure without doing anything.
2403 * @qm: The qm needed to be fronzen.
2404 *
2405 * This function frozes QM, then we can do SRIOV disabling.
2406 */
2407static int qm_frozen(struct hisi_qm *qm)
2408{
2409        if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))
2410                return 0;
2411
2412        down_write(&qm->qps_lock);
2413
2414        if (!qm->qp_in_used) {
2415                qm->qp_in_used = qm->qp_num;
2416                up_write(&qm->qps_lock);
2417                set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);
2418                return 0;
2419        }
2420
2421        up_write(&qm->qps_lock);
2422
2423        return -EBUSY;
2424}
2425
2426static int qm_try_frozen_vfs(struct pci_dev *pdev,
2427                             struct hisi_qm_list *qm_list)
2428{
2429        struct hisi_qm *qm, *vf_qm;
2430        struct pci_dev *dev;
2431        int ret = 0;
2432
2433        if (!qm_list || !pdev)
2434                return -EINVAL;
2435
2436        /* Try to frozen all the VFs as disable SRIOV */
2437        mutex_lock(&qm_list->lock);
2438        list_for_each_entry(qm, &qm_list->list, list) {
2439                dev = qm->pdev;
2440                if (dev == pdev)
2441                        continue;
2442                if (pci_physfn(dev) == pdev) {
2443                        vf_qm = pci_get_drvdata(dev);
2444                        ret = qm_frozen(vf_qm);
2445                        if (ret)
2446                                goto frozen_fail;
2447                }
2448        }
2449
2450frozen_fail:
2451        mutex_unlock(&qm_list->lock);
2452
2453        return ret;
2454}
2455
2456/**
2457 * hisi_qm_wait_task_finish() - Wait until the task is finished
2458 * when removing the driver.
2459 * @qm: The qm needed to wait for the task to finish.
2460 * @qm_list: The list of all available devices.
2461 */
2462void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
2463{
2464        while (qm_frozen(qm) ||
2465               ((qm->fun_type == QM_HW_PF) &&
2466               qm_try_frozen_vfs(qm->pdev, qm_list))) {
2467                msleep(WAIT_PERIOD);
2468        }
2469
2470        while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||
2471               test_bit(QM_RESETTING, &qm->misc_ctl))
2472                msleep(WAIT_PERIOD);
2473
2474        udelay(REMOVE_WAIT_DELAY);
2475}
2476EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
2477
2478/**
2479 * hisi_qm_get_free_qp_num() - Get free number of qp in qm.
2480 * @qm: The qm which want to get free qp.
2481 *
2482 * This function return free number of qp in qm.
2483 */
2484int hisi_qm_get_free_qp_num(struct hisi_qm *qm)
2485{
2486        int ret;
2487
2488        down_read(&qm->qps_lock);
2489        ret = qm->qp_num - qm->qp_in_used;
2490        up_read(&qm->qps_lock);
2491
2492        return ret;
2493}
2494EXPORT_SYMBOL_GPL(hisi_qm_get_free_qp_num);
2495
2496static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
2497{
2498        struct device *dev = &qm->pdev->dev;
2499        struct qm_dma *qdma;
2500        int i;
2501
2502        for (i = num - 1; i >= 0; i--) {
2503                qdma = &qm->qp_array[i].qdma;
2504                dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
2505        }
2506
2507        kfree(qm->qp_array);
2508}
2509
2510static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id)
2511{
2512        struct device *dev = &qm->pdev->dev;
2513        size_t off = qm->sqe_size * QM_Q_DEPTH;
2514        struct hisi_qp *qp;
2515
2516        qp = &qm->qp_array[id];
2517        qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
2518                                         GFP_KERNEL);
2519        if (!qp->qdma.va)
2520                return -ENOMEM;
2521
2522        qp->sqe = qp->qdma.va;
2523        qp->sqe_dma = qp->qdma.dma;
2524        qp->cqe = qp->qdma.va + off;
2525        qp->cqe_dma = qp->qdma.dma + off;
2526        qp->qdma.size = dma_size;
2527        qp->qm = qm;
2528        qp->qp_id = id;
2529
2530        return 0;
2531}
2532
2533static int hisi_qm_memory_init(struct hisi_qm *qm)
2534{
2535        struct device *dev = &qm->pdev->dev;
2536        size_t qp_dma_size, off = 0;
2537        int i, ret = 0;
2538
2539#define QM_INIT_BUF(qm, type, num) do { \
2540        (qm)->type = ((qm)->qdma.va + (off)); \
2541        (qm)->type##_dma = (qm)->qdma.dma + (off); \
2542        off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
2543} while (0)
2544
2545        idr_init(&qm->qp_idr);
2546        qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) +
2547                        QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
2548                        QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
2549                        QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
2550        qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
2551                                         GFP_ATOMIC);
2552        dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
2553        if (!qm->qdma.va)
2554                return -ENOMEM;
2555
2556        QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH);
2557        QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
2558        QM_INIT_BUF(qm, sqc, qm->qp_num);
2559        QM_INIT_BUF(qm, cqc, qm->qp_num);
2560
2561        qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
2562        if (!qm->qp_array) {
2563                ret = -ENOMEM;
2564                goto err_alloc_qp_array;
2565        }
2566
2567        /* one more page for device or qp statuses */
2568        qp_dma_size = qm->sqe_size * QM_Q_DEPTH +
2569                      sizeof(struct qm_cqe) * QM_Q_DEPTH;
2570        qp_dma_size = PAGE_ALIGN(qp_dma_size);
2571        for (i = 0; i < qm->qp_num; i++) {
2572                ret = hisi_qp_memory_init(qm, qp_dma_size, i);
2573                if (ret)
2574                        goto err_init_qp_mem;
2575
2576                dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
2577        }
2578
2579        return ret;
2580
2581err_init_qp_mem:
2582        hisi_qp_memory_uninit(qm, i);
2583err_alloc_qp_array:
2584        dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
2585
2586        return ret;
2587}
2588
2589static void hisi_qm_pre_init(struct hisi_qm *qm)
2590{
2591        struct pci_dev *pdev = qm->pdev;
2592
2593        if (qm->ver == QM_HW_V1)
2594                qm->ops = &qm_hw_ops_v1;
2595        else if (qm->ver == QM_HW_V2)
2596                qm->ops = &qm_hw_ops_v2;
2597        else
2598                qm->ops = &qm_hw_ops_v3;
2599
2600        pci_set_drvdata(pdev, qm);
2601        mutex_init(&qm->mailbox_lock);
2602        init_rwsem(&qm->qps_lock);
2603        qm->qp_in_used = 0;
2604        qm->misc_ctl = false;
2605}
2606
2607static void qm_put_pci_res(struct hisi_qm *qm)
2608{
2609        struct pci_dev *pdev = qm->pdev;
2610
2611        if (qm->use_db_isolation)
2612                iounmap(qm->db_io_base);
2613
2614        iounmap(qm->io_base);
2615        pci_release_mem_regions(pdev);
2616}
2617
2618static void hisi_qm_pci_uninit(struct hisi_qm *qm)
2619{
2620        struct pci_dev *pdev = qm->pdev;
2621
2622        pci_free_irq_vectors(pdev);
2623        qm_put_pci_res(qm);
2624        pci_disable_device(pdev);
2625}
2626
2627/**
2628 * hisi_qm_uninit() - Uninitialize qm.
2629 * @qm: The qm needed uninit.
2630 *
2631 * This function uninits qm related device resources.
2632 */
2633void hisi_qm_uninit(struct hisi_qm *qm)
2634{
2635        struct pci_dev *pdev = qm->pdev;
2636        struct device *dev = &pdev->dev;
2637
2638        down_write(&qm->qps_lock);
2639
2640        if (!qm_avail_state(qm, QM_CLOSE)) {
2641                up_write(&qm->qps_lock);
2642                return;
2643        }
2644
2645        hisi_qp_memory_uninit(qm, qm->qp_num);
2646        idr_destroy(&qm->qp_idr);
2647
2648        if (qm->qdma.va) {
2649                hisi_qm_cache_wb(qm);
2650                dma_free_coherent(dev, qm->qdma.size,
2651                                  qm->qdma.va, qm->qdma.dma);
2652        }
2653
2654        qm_irq_unregister(qm);
2655        hisi_qm_pci_uninit(qm);
2656        uacce_remove(qm->uacce);
2657        qm->uacce = NULL;
2658
2659        up_write(&qm->qps_lock);
2660}
2661EXPORT_SYMBOL_GPL(hisi_qm_uninit);
2662
2663/**
2664 * hisi_qm_get_vft() - Get vft from a qm.
2665 * @qm: The qm we want to get its vft.
2666 * @base: The base number of queue in vft.
2667 * @number: The number of queues in vft.
2668 *
2669 * We can allocate multiple queues to a qm by configuring virtual function
2670 * table. We get related configures by this function. Normally, we call this
2671 * function in VF driver to get the queue information.
2672 *
2673 * qm hw v1 does not support this interface.
2674 */
2675int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
2676{
2677        if (!base || !number)
2678                return -EINVAL;
2679
2680        if (!qm->ops->get_vft) {
2681                dev_err(&qm->pdev->dev, "Don't support vft read!\n");
2682                return -EINVAL;
2683        }
2684
2685        return qm->ops->get_vft(qm, base, number);
2686}
2687EXPORT_SYMBOL_GPL(hisi_qm_get_vft);
2688
2689/**
2690 * hisi_qm_set_vft() - Set vft to a qm.
2691 * @qm: The qm we want to set its vft.
2692 * @fun_num: The function number.
2693 * @base: The base number of queue in vft.
2694 * @number: The number of queues in vft.
2695 *
2696 * This function is alway called in PF driver, it is used to assign queues
2697 * among PF and VFs.
2698 *
2699 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
2700 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
2701 * (VF function number 0x2)
2702 */
2703static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
2704                    u32 number)
2705{
2706        u32 max_q_num = qm->ctrl_qp_num;
2707
2708        if (base >= max_q_num || number > max_q_num ||
2709            (base + number) > max_q_num)
2710                return -EINVAL;
2711
2712        return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
2713}
2714
2715static void qm_init_eq_aeq_status(struct hisi_qm *qm)
2716{
2717        struct hisi_qm_status *status = &qm->status;
2718
2719        status->eq_head = 0;
2720        status->aeq_head = 0;
2721        status->eqc_phase = true;
2722        status->aeqc_phase = true;
2723}
2724
2725static int qm_eq_ctx_cfg(struct hisi_qm *qm)
2726{
2727        struct device *dev = &qm->pdev->dev;
2728        struct qm_eqc *eqc;
2729        dma_addr_t eqc_dma;
2730        int ret;
2731
2732        eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
2733        if (!eqc)
2734                return -ENOMEM;
2735
2736        eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
2737        eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
2738        if (qm->ver == QM_HW_V1)
2739                eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
2740        eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
2741
2742        eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
2743                                 DMA_TO_DEVICE);
2744        if (dma_mapping_error(dev, eqc_dma)) {
2745                kfree(eqc);
2746                return -ENOMEM;
2747        }
2748
2749        ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
2750        dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
2751        kfree(eqc);
2752
2753        return ret;
2754}
2755
2756static int qm_aeq_ctx_cfg(struct hisi_qm *qm)
2757{
2758        struct device *dev = &qm->pdev->dev;
2759        struct qm_aeqc *aeqc;
2760        dma_addr_t aeqc_dma;
2761        int ret;
2762
2763        aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
2764        if (!aeqc)
2765                return -ENOMEM;
2766
2767        aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
2768        aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
2769        aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
2770
2771        aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
2772                                  DMA_TO_DEVICE);
2773        if (dma_mapping_error(dev, aeqc_dma)) {
2774                kfree(aeqc);
2775                return -ENOMEM;
2776        }
2777
2778        ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
2779        dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
2780        kfree(aeqc);
2781
2782        return ret;
2783}
2784
2785static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)
2786{
2787        struct device *dev = &qm->pdev->dev;
2788        int ret;
2789
2790        qm_init_eq_aeq_status(qm);
2791
2792        ret = qm_eq_ctx_cfg(qm);
2793        if (ret) {
2794                dev_err(dev, "Set eqc failed!\n");
2795                return ret;
2796        }
2797
2798        return qm_aeq_ctx_cfg(qm);
2799}
2800
2801static int __hisi_qm_start(struct hisi_qm *qm)
2802{
2803        int ret;
2804
2805        WARN_ON(!qm->qdma.va);
2806
2807        if (qm->fun_type == QM_HW_PF) {
2808                ret = qm_dev_mem_reset(qm);
2809                if (ret)
2810                        return ret;
2811
2812                ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
2813                if (ret)
2814                        return ret;
2815        }
2816
2817        ret = qm_eq_aeq_ctx_cfg(qm);
2818        if (ret)
2819                return ret;
2820
2821        ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
2822        if (ret)
2823                return ret;
2824
2825        ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
2826        if (ret)
2827                return ret;
2828
2829        writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
2830        writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
2831
2832        return 0;
2833}
2834
2835/**
2836 * hisi_qm_start() - start qm
2837 * @qm: The qm to be started.
2838 *
2839 * This function starts a qm, then we can allocate qp from this qm.
2840 */
2841int hisi_qm_start(struct hisi_qm *qm)
2842{
2843        struct device *dev = &qm->pdev->dev;
2844        int ret = 0;
2845
2846        down_write(&qm->qps_lock);
2847
2848        if (!qm_avail_state(qm, QM_START)) {
2849                up_write(&qm->qps_lock);
2850                return -EPERM;
2851        }
2852
2853        dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
2854
2855        if (!qm->qp_num) {
2856                dev_err(dev, "qp_num should not be 0\n");
2857                ret = -EINVAL;
2858                goto err_unlock;
2859        }
2860
2861        ret = __hisi_qm_start(qm);
2862        if (!ret)
2863                atomic_set(&qm->status.flags, QM_START);
2864
2865err_unlock:
2866        up_write(&qm->qps_lock);
2867        return ret;
2868}
2869EXPORT_SYMBOL_GPL(hisi_qm_start);
2870
2871static int qm_restart(struct hisi_qm *qm)
2872{
2873        struct device *dev = &qm->pdev->dev;
2874        struct hisi_qp *qp;
2875        int ret, i;
2876
2877        ret = hisi_qm_start(qm);
2878        if (ret < 0)
2879                return ret;
2880
2881        down_write(&qm->qps_lock);
2882        for (i = 0; i < qm->qp_num; i++) {
2883                qp = &qm->qp_array[i];
2884                if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
2885                    qp->is_resetting == true) {
2886                        ret = qm_start_qp_nolock(qp, 0);
2887                        if (ret < 0) {
2888                                dev_err(dev, "Failed to start qp%d!\n", i);
2889
2890                                up_write(&qm->qps_lock);
2891                                return ret;
2892                        }
2893                        qp->is_resetting = false;
2894                }
2895        }
2896        up_write(&qm->qps_lock);
2897
2898        return 0;
2899}
2900
2901/* Stop started qps in reset flow */
2902static int qm_stop_started_qp(struct hisi_qm *qm)
2903{
2904        struct device *dev = &qm->pdev->dev;
2905        struct hisi_qp *qp;
2906        int i, ret;
2907
2908        for (i = 0; i < qm->qp_num; i++) {
2909                qp = &qm->qp_array[i];
2910                if (qp && atomic_read(&qp->qp_status.flags) == QP_START) {
2911                        qp->is_resetting = true;
2912                        ret = qm_stop_qp_nolock(qp);
2913                        if (ret < 0) {
2914                                dev_err(dev, "Failed to stop qp%d!\n", i);
2915                                return ret;
2916                        }
2917                }
2918        }
2919
2920        return 0;
2921}
2922
2923
2924/**
2925 * qm_clear_queues() - Clear all queues memory in a qm.
2926 * @qm: The qm in which the queues will be cleared.
2927 *
2928 * This function clears all queues memory in a qm. Reset of accelerator can
2929 * use this to clear queues.
2930 */
2931static void qm_clear_queues(struct hisi_qm *qm)
2932{
2933        struct hisi_qp *qp;
2934        int i;
2935
2936        for (i = 0; i < qm->qp_num; i++) {
2937                qp = &qm->qp_array[i];
2938                if (qp->is_resetting)
2939                        memset(qp->qdma.va, 0, qp->qdma.size);
2940        }
2941
2942        memset(qm->qdma.va, 0, qm->qdma.size);
2943}
2944
2945/**
2946 * hisi_qm_stop() - Stop a qm.
2947 * @qm: The qm which will be stopped.
2948 * @r: The reason to stop qm.
2949 *
2950 * This function stops qm and its qps, then qm can not accept request.
2951 * Related resources are not released at this state, we can use hisi_qm_start
2952 * to let qm start again.
2953 */
2954int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
2955{
2956        struct device *dev = &qm->pdev->dev;
2957        int ret = 0;
2958
2959        down_write(&qm->qps_lock);
2960
2961        qm->status.stop_reason = r;
2962        if (!qm_avail_state(qm, QM_STOP)) {
2963                ret = -EPERM;
2964                goto err_unlock;
2965        }
2966
2967        if (qm->status.stop_reason == QM_SOFT_RESET ||
2968            qm->status.stop_reason == QM_FLR) {
2969                ret = qm_stop_started_qp(qm);
2970                if (ret < 0) {
2971                        dev_err(dev, "Failed to stop started qp!\n");
2972                        goto err_unlock;
2973                }
2974        }
2975
2976        /* Mask eq and aeq irq */
2977        writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
2978        writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
2979
2980        if (qm->fun_type == QM_HW_PF) {
2981                ret = hisi_qm_set_vft(qm, 0, 0, 0);
2982                if (ret < 0) {
2983                        dev_err(dev, "Failed to set vft!\n");
2984                        ret = -EBUSY;
2985                        goto err_unlock;
2986                }
2987        }
2988
2989        qm_clear_queues(qm);
2990        atomic_set(&qm->status.flags, QM_STOP);
2991
2992err_unlock:
2993        up_write(&qm->qps_lock);
2994        return ret;
2995}
2996EXPORT_SYMBOL_GPL(hisi_qm_stop);
2997
2998static ssize_t qm_status_read(struct file *filp, char __user *buffer,
2999                              size_t count, loff_t *pos)
3000{
3001        struct hisi_qm *qm = filp->private_data;
3002        char buf[QM_DBG_READ_LEN];
3003        int val, len;
3004
3005        val = atomic_read(&qm->status.flags);
3006        len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n", qm_s[val]);
3007
3008        return simple_read_from_buffer(buffer, count, pos, buf, len);
3009}
3010
3011static const struct file_operations qm_status_fops = {
3012        .owner = THIS_MODULE,
3013        .open = simple_open,
3014        .read = qm_status_read,
3015};
3016
3017static int qm_debugfs_atomic64_set(void *data, u64 val)
3018{
3019        if (val)
3020                return -EINVAL;
3021
3022        atomic64_set((atomic64_t *)data, 0);
3023
3024        return 0;
3025}
3026
3027static int qm_debugfs_atomic64_get(void *data, u64 *val)
3028{
3029        *val = atomic64_read((atomic64_t *)data);
3030
3031        return 0;
3032}
3033
3034DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get,
3035                         qm_debugfs_atomic64_set, "%llu\n");
3036
3037/**
3038 * hisi_qm_debug_init() - Initialize qm related debugfs files.
3039 * @qm: The qm for which we want to add debugfs files.
3040 *
3041 * Create qm related debugfs files.
3042 */
3043void hisi_qm_debug_init(struct hisi_qm *qm)
3044{
3045        struct qm_dfx *dfx = &qm->debug.dfx;
3046        struct dentry *qm_d;
3047        void *data;
3048        int i;
3049
3050        qm_d = debugfs_create_dir("qm", qm->debug.debug_root);
3051        qm->debug.qm_d = qm_d;
3052
3053        /* only show this in PF */
3054        if (qm->fun_type == QM_HW_PF) {
3055                qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM);
3056                for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
3057                        qm_create_debugfs_file(qm, qm_d, i);
3058        }
3059
3060        debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops);
3061
3062        debugfs_create_file("cmd", 0444, qm->debug.qm_d, qm, &qm_cmd_fops);
3063
3064        debugfs_create_file("status", 0444, qm->debug.qm_d, qm,
3065                        &qm_status_fops);
3066        for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) {
3067                data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset);
3068                debugfs_create_file(qm_dfx_files[i].name,
3069                        0644,
3070                        qm_d,
3071                        data,
3072                        &qm_atomic64_ops);
3073        }
3074}
3075EXPORT_SYMBOL_GPL(hisi_qm_debug_init);
3076
3077/**
3078 * hisi_qm_debug_regs_clear() - clear qm debug related registers.
3079 * @qm: The qm for which we want to clear its debug registers.
3080 */
3081void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
3082{
3083        struct qm_dfx_registers *regs;
3084        int i;
3085
3086        /* clear current_qm */
3087        writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
3088        writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
3089
3090        /* clear current_q */
3091        writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
3092        writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
3093
3094        /*
3095         * these registers are reading and clearing, so clear them after
3096         * reading them.
3097         */
3098        writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE);
3099
3100        regs = qm_dfx_regs;
3101        for (i = 0; i < CNT_CYC_REGS_NUM; i++) {
3102                readl(qm->io_base + regs->reg_offset);
3103                regs++;
3104        }
3105
3106        writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE);
3107}
3108EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
3109
3110static void qm_hw_error_init(struct hisi_qm *qm)
3111{
3112        struct hisi_qm_err_info *err_info = &qm->err_info;
3113
3114        if (!qm->ops->hw_error_init) {
3115                dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
3116                return;
3117        }
3118
3119        qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, err_info->fe);
3120}
3121
3122static void qm_hw_error_uninit(struct hisi_qm *qm)
3123{
3124        if (!qm->ops->hw_error_uninit) {
3125                dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
3126                return;
3127        }
3128
3129        qm->ops->hw_error_uninit(qm);
3130}
3131
3132static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
3133{
3134        if (!qm->ops->hw_error_handle) {
3135                dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
3136                return ACC_ERR_NONE;
3137        }
3138
3139        return qm->ops->hw_error_handle(qm);
3140}
3141
3142/**
3143 * hisi_qm_dev_err_init() - Initialize device error configuration.
3144 * @qm: The qm for which we want to do error initialization.
3145 *
3146 * Initialize QM and device error related configuration.
3147 */
3148void hisi_qm_dev_err_init(struct hisi_qm *qm)
3149{
3150        if (qm->fun_type == QM_HW_VF)
3151                return;
3152
3153        qm_hw_error_init(qm);
3154
3155        if (!qm->err_ini->hw_err_enable) {
3156                dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
3157                return;
3158        }
3159        qm->err_ini->hw_err_enable(qm);
3160}
3161EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
3162
3163/**
3164 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
3165 * @qm: The qm for which we want to do error uninitialization.
3166 *
3167 * Uninitialize QM and device error related configuration.
3168 */
3169void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
3170{
3171        if (qm->fun_type == QM_HW_VF)
3172                return;
3173
3174        qm_hw_error_uninit(qm);
3175
3176        if (!qm->err_ini->hw_err_disable) {
3177                dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
3178                return;
3179        }
3180        qm->err_ini->hw_err_disable(qm);
3181}
3182EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
3183
3184/**
3185 * hisi_qm_free_qps() - free multiple queue pairs.
3186 * @qps: The queue pairs need to be freed.
3187 * @qp_num: The num of queue pairs.
3188 */
3189void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
3190{
3191        int i;
3192
3193        if (!qps || qp_num <= 0)
3194                return;
3195
3196        for (i = qp_num - 1; i >= 0; i--)
3197                hisi_qm_release_qp(qps[i]);
3198}
3199EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
3200
3201static void free_list(struct list_head *head)
3202{
3203        struct hisi_qm_resource *res, *tmp;
3204
3205        list_for_each_entry_safe(res, tmp, head, list) {
3206                list_del(&res->list);
3207                kfree(res);
3208        }
3209}
3210
3211static int hisi_qm_sort_devices(int node, struct list_head *head,
3212                                struct hisi_qm_list *qm_list)
3213{
3214        struct hisi_qm_resource *res, *tmp;
3215        struct hisi_qm *qm;
3216        struct list_head *n;
3217        struct device *dev;
3218        int dev_node = 0;
3219
3220        list_for_each_entry(qm, &qm_list->list, list) {
3221                dev = &qm->pdev->dev;
3222
3223                if (IS_ENABLED(CONFIG_NUMA)) {
3224                        dev_node = dev_to_node(dev);
3225                        if (dev_node < 0)
3226                                dev_node = 0;
3227                }
3228
3229                res = kzalloc(sizeof(*res), GFP_KERNEL);
3230                if (!res)
3231                        return -ENOMEM;
3232
3233                res->qm = qm;
3234                res->distance = node_distance(dev_node, node);
3235                n = head;
3236                list_for_each_entry(tmp, head, list) {
3237                        if (res->distance < tmp->distance) {
3238                                n = &tmp->list;
3239                                break;
3240                        }
3241                }
3242                list_add_tail(&res->list, n);
3243        }
3244
3245        return 0;
3246}
3247
3248/**
3249 * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
3250 * @qm_list: The list of all available devices.
3251 * @qp_num: The number of queue pairs need created.
3252 * @alg_type: The algorithm type.
3253 * @node: The numa node.
3254 * @qps: The queue pairs need created.
3255 *
3256 * This function will sort all available device according to numa distance.
3257 * Then try to create all queue pairs from one device, if all devices do
3258 * not meet the requirements will return error.
3259 */
3260int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
3261                           u8 alg_type, int node, struct hisi_qp **qps)
3262{
3263        struct hisi_qm_resource *tmp;
3264        int ret = -ENODEV;
3265        LIST_HEAD(head);
3266        int i;
3267
3268        if (!qps || !qm_list || qp_num <= 0)
3269                return -EINVAL;
3270
3271        mutex_lock(&qm_list->lock);
3272        if (hisi_qm_sort_devices(node, &head, qm_list)) {
3273                mutex_unlock(&qm_list->lock);
3274                goto err;
3275        }
3276
3277        list_for_each_entry(tmp, &head, list) {
3278                for (i = 0; i < qp_num; i++) {
3279                        qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
3280                        if (IS_ERR(qps[i])) {
3281                                hisi_qm_free_qps(qps, i);
3282                                break;
3283                        }
3284                }
3285
3286                if (i == qp_num) {
3287                        ret = 0;
3288                        break;
3289                }
3290        }
3291
3292        mutex_unlock(&qm_list->lock);
3293        if (ret)
3294                pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",
3295                        node, alg_type, qp_num);
3296
3297err:
3298        free_list(&head);
3299        return ret;
3300}
3301EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
3302
3303static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
3304{
3305        u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
3306        u32 max_qp_num = qm->max_qp_num;
3307        u32 q_base = qm->qp_num;
3308        int ret;
3309
3310        if (!num_vfs)
3311                return -EINVAL;
3312
3313        vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
3314
3315        /* If vfs_q_num is less than num_vfs, return error. */
3316        if (vfs_q_num < num_vfs)
3317                return -EINVAL;
3318
3319        q_num = vfs_q_num / num_vfs;
3320        remain_q_num = vfs_q_num % num_vfs;
3321
3322        for (i = num_vfs; i > 0; i--) {
3323                /*
3324                 * if q_num + remain_q_num > max_qp_num in last vf, divide the
3325                 * remaining queues equally.
3326                 */
3327                if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
3328                        act_q_num = q_num + remain_q_num;
3329                        remain_q_num = 0;
3330                } else if (remain_q_num > 0) {
3331                        act_q_num = q_num + 1;
3332                        remain_q_num--;
3333                } else {
3334                        act_q_num = q_num;
3335                }
3336
3337                act_q_num = min_t(int, act_q_num, max_qp_num);
3338                ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
3339                if (ret) {
3340                        for (j = num_vfs; j > i; j--)
3341                                hisi_qm_set_vft(qm, j, 0, 0);
3342                        return ret;
3343                }
3344                q_base += act_q_num;
3345        }
3346
3347        return 0;
3348}
3349
3350static int qm_clear_vft_config(struct hisi_qm *qm)
3351{
3352        int ret;
3353        u32 i;
3354
3355        for (i = 1; i <= qm->vfs_num; i++) {
3356                ret = hisi_qm_set_vft(qm, i, 0, 0);
3357                if (ret)
3358                        return ret;
3359        }
3360        qm->vfs_num = 0;
3361
3362        return 0;
3363}
3364
3365/**
3366 * hisi_qm_sriov_enable() - enable virtual functions
3367 * @pdev: the PCIe device
3368 * @max_vfs: the number of virtual functions to enable
3369 *
3370 * Returns the number of enabled VFs. If there are VFs enabled already or
3371 * max_vfs is more than the total number of device can be enabled, returns
3372 * failure.
3373 */
3374int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
3375{
3376        struct hisi_qm *qm = pci_get_drvdata(pdev);
3377        int pre_existing_vfs, num_vfs, total_vfs, ret;
3378
3379        total_vfs = pci_sriov_get_totalvfs(pdev);
3380        pre_existing_vfs = pci_num_vf(pdev);
3381        if (pre_existing_vfs) {
3382                pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
3383                        pre_existing_vfs);
3384                return 0;
3385        }
3386
3387        num_vfs = min_t(int, max_vfs, total_vfs);
3388        ret = qm_vf_q_assign(qm, num_vfs);
3389        if (ret) {
3390                pci_err(pdev, "Can't assign queues for VF!\n");
3391                return ret;
3392        }
3393
3394        qm->vfs_num = num_vfs;
3395
3396        ret = pci_enable_sriov(pdev, num_vfs);
3397        if (ret) {
3398                pci_err(pdev, "Can't enable VF!\n");
3399                qm_clear_vft_config(qm);
3400                return ret;
3401        }
3402
3403        pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
3404
3405        return num_vfs;
3406}
3407EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
3408
3409/**
3410 * hisi_qm_sriov_disable - disable virtual functions
3411 * @pdev: the PCI device.
3412 * @is_frozen: true when all the VFs are frozen.
3413 *
3414 * Return failure if there are VFs assigned already or VF is in used.
3415 */
3416int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
3417{
3418        struct hisi_qm *qm = pci_get_drvdata(pdev);
3419
3420        if (pci_vfs_assigned(pdev)) {
3421                pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
3422                return -EPERM;
3423        }
3424
3425        /* While VF is in used, SRIOV cannot be disabled. */
3426        if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
3427                pci_err(pdev, "Task is using its VF!\n");
3428                return -EBUSY;
3429        }
3430
3431        pci_disable_sriov(pdev);
3432        return qm_clear_vft_config(qm);
3433}
3434EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
3435
3436/**
3437 * hisi_qm_sriov_configure - configure the number of VFs
3438 * @pdev: The PCI device
3439 * @num_vfs: The number of VFs need enabled
3440 *
3441 * Enable SR-IOV according to num_vfs, 0 means disable.
3442 */
3443int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
3444{
3445        if (num_vfs == 0)
3446                return hisi_qm_sriov_disable(pdev, false);
3447        else
3448                return hisi_qm_sriov_enable(pdev, num_vfs);
3449}
3450EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
3451
3452static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
3453{
3454        u32 err_sts;
3455
3456        if (!qm->err_ini->get_dev_hw_err_status) {
3457                dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n");
3458                return ACC_ERR_NONE;
3459        }
3460
3461        /* get device hardware error status */
3462        err_sts = qm->err_ini->get_dev_hw_err_status(qm);
3463        if (err_sts) {
3464                if (err_sts & qm->err_info.ecc_2bits_mask)
3465                        qm->err_status.is_dev_ecc_mbit = true;
3466
3467                if (qm->err_ini->log_dev_hw_err)
3468                        qm->err_ini->log_dev_hw_err(qm, err_sts);
3469
3470                /* ce error does not need to be reset */
3471                if ((err_sts | qm->err_info.dev_ce_mask) ==
3472                     qm->err_info.dev_ce_mask) {
3473                        if (qm->err_ini->clear_dev_hw_err_status)
3474                                qm->err_ini->clear_dev_hw_err_status(qm,
3475                                                                err_sts);
3476
3477                        return ACC_ERR_RECOVERED;
3478                }
3479
3480                return ACC_ERR_NEED_RESET;
3481        }
3482
3483        return ACC_ERR_RECOVERED;
3484}
3485
3486static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
3487{
3488        enum acc_err_result qm_ret, dev_ret;
3489
3490        /* log qm error */
3491        qm_ret = qm_hw_error_handle(qm);
3492
3493        /* log device error */
3494        dev_ret = qm_dev_err_handle(qm);
3495
3496        return (qm_ret == ACC_ERR_NEED_RESET ||
3497                dev_ret == ACC_ERR_NEED_RESET) ?
3498                ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
3499}
3500
3501/**
3502 * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
3503 * @pdev: The PCI device which need report error.
3504 * @state: The connectivity between CPU and device.
3505 *
3506 * We register this function into PCIe AER handlers, It will report device or
3507 * qm hardware error status when error occur.
3508 */
3509pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
3510                                          pci_channel_state_t state)
3511{
3512        struct hisi_qm *qm = pci_get_drvdata(pdev);
3513        enum acc_err_result ret;
3514
3515        if (pdev->is_virtfn)
3516                return PCI_ERS_RESULT_NONE;
3517
3518        pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);
3519        if (state == pci_channel_io_perm_failure)
3520                return PCI_ERS_RESULT_DISCONNECT;
3521
3522        ret = qm_process_dev_error(qm);
3523        if (ret == ACC_ERR_NEED_RESET)
3524                return PCI_ERS_RESULT_NEED_RESET;
3525
3526        return PCI_ERS_RESULT_RECOVERED;
3527}
3528EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
3529
3530static u32 qm_get_hw_error_status(struct hisi_qm *qm)
3531{
3532        return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
3533}
3534
3535static int qm_check_req_recv(struct hisi_qm *qm)
3536{
3537        struct pci_dev *pdev = qm->pdev;
3538        int ret;
3539        u32 val;
3540
3541        writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
3542        ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
3543                                         (val == ACC_VENDOR_ID_VALUE),
3544                                         POLL_PERIOD, POLL_TIMEOUT);
3545        if (ret) {
3546                dev_err(&pdev->dev, "Fails to read QM reg!\n");
3547                return ret;
3548        }
3549
3550        writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
3551        ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
3552                                         (val == PCI_VENDOR_ID_HUAWEI),
3553                                         POLL_PERIOD, POLL_TIMEOUT);
3554        if (ret)
3555                dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
3556
3557        return ret;
3558}
3559
3560static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
3561{
3562        struct pci_dev *pdev = qm->pdev;
3563        u16 cmd;
3564        int i;
3565
3566        pci_read_config_word(pdev, PCI_COMMAND, &cmd);
3567        if (set)
3568                cmd |= PCI_COMMAND_MEMORY;
3569        else
3570                cmd &= ~PCI_COMMAND_MEMORY;
3571
3572        pci_write_config_word(pdev, PCI_COMMAND, cmd);
3573        for (i = 0; i < MAX_WAIT_COUNTS; i++) {
3574                pci_read_config_word(pdev, PCI_COMMAND, &cmd);
3575                if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
3576                        return 0;
3577
3578                udelay(1);
3579        }
3580
3581        return -ETIMEDOUT;
3582}
3583
3584static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
3585{
3586        struct pci_dev *pdev = qm->pdev;
3587        u16 sriov_ctrl;
3588        int pos;
3589        int i;
3590
3591        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
3592        pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
3593        if (set)
3594                sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
3595        else
3596                sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
3597        pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
3598
3599        for (i = 0; i < MAX_WAIT_COUNTS; i++) {
3600                pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
3601                if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
3602                    ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
3603                        return 0;
3604
3605                udelay(1);
3606        }
3607
3608        return -ETIMEDOUT;
3609}
3610
3611static int qm_set_msi(struct hisi_qm *qm, bool set)
3612{
3613        struct pci_dev *pdev = qm->pdev;
3614
3615        if (set) {
3616                pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
3617                                       0);
3618        } else {
3619                pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
3620                                       ACC_PEH_MSI_DISABLE);
3621                if (qm->err_status.is_qm_ecc_mbit ||
3622                    qm->err_status.is_dev_ecc_mbit)
3623                        return 0;
3624
3625                mdelay(1);
3626                if (readl(qm->io_base + QM_PEH_DFX_INFO0))
3627                        return -EFAULT;
3628        }
3629
3630        return 0;
3631}
3632
3633static int qm_vf_reset_prepare(struct hisi_qm *qm,
3634                               enum qm_stop_reason stop_reason)
3635{
3636        struct hisi_qm_list *qm_list = qm->qm_list;
3637        struct pci_dev *pdev = qm->pdev;
3638        struct pci_dev *virtfn;
3639        struct hisi_qm *vf_qm;
3640        int ret = 0;
3641
3642        mutex_lock(&qm_list->lock);
3643        list_for_each_entry(vf_qm, &qm_list->list, list) {
3644                virtfn = vf_qm->pdev;
3645                if (virtfn == pdev)
3646                        continue;
3647
3648                if (pci_physfn(virtfn) == pdev) {
3649                        /* save VFs PCIE BAR configuration */
3650                        pci_save_state(virtfn);
3651
3652                        ret = hisi_qm_stop(vf_qm, stop_reason);
3653                        if (ret)
3654                                goto stop_fail;
3655                }
3656        }
3657
3658stop_fail:
3659        mutex_unlock(&qm_list->lock);
3660        return ret;
3661}
3662
3663static int qm_reset_prepare_ready(struct hisi_qm *qm)
3664{
3665        struct pci_dev *pdev = qm->pdev;
3666        struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
3667        int delay = 0;
3668
3669        /* All reset requests need to be queued for processing */
3670        while (test_and_set_bit(QM_RESETTING, &pf_qm->misc_ctl)) {
3671                msleep(++delay);
3672                if (delay > QM_RESET_WAIT_TIMEOUT)
3673                        return -EBUSY;
3674        }
3675
3676        return 0;
3677}
3678
3679static int qm_controller_reset_prepare(struct hisi_qm *qm)
3680{
3681        struct pci_dev *pdev = qm->pdev;
3682        int ret;
3683
3684        ret = qm_reset_prepare_ready(qm);
3685        if (ret) {
3686                pci_err(pdev, "Controller reset not ready!\n");
3687                return ret;
3688        }
3689
3690        if (qm->vfs_num) {
3691                ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET);
3692                if (ret) {
3693                        pci_err(pdev, "Fails to stop VFs!\n");
3694                        clear_bit(QM_RESETTING, &qm->misc_ctl);
3695                        return ret;
3696                }
3697        }
3698
3699        ret = hisi_qm_stop(qm, QM_SOFT_RESET);
3700        if (ret) {
3701                pci_err(pdev, "Fails to stop QM!\n");
3702                clear_bit(QM_RESETTING, &qm->misc_ctl);
3703                return ret;
3704        }
3705
3706        clear_bit(QM_RST_SCHED, &qm->misc_ctl);
3707
3708        return 0;
3709}
3710
3711static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
3712{
3713        u32 nfe_enb = 0;
3714
3715        if (!qm->err_status.is_dev_ecc_mbit &&
3716            qm->err_status.is_qm_ecc_mbit &&
3717            qm->err_ini->close_axi_master_ooo) {
3718
3719                qm->err_ini->close_axi_master_ooo(qm);
3720
3721        } else if (qm->err_status.is_dev_ecc_mbit &&
3722                   !qm->err_status.is_qm_ecc_mbit &&
3723                   !qm->err_ini->close_axi_master_ooo) {
3724
3725                nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
3726                writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
3727                       qm->io_base + QM_RAS_NFE_ENABLE);
3728                writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
3729        }
3730}
3731
3732static int qm_soft_reset(struct hisi_qm *qm)
3733{
3734        struct pci_dev *pdev = qm->pdev;
3735        int ret;
3736        u32 val;
3737
3738        /* Ensure all doorbells and mailboxes received by QM */
3739        ret = qm_check_req_recv(qm);
3740        if (ret)
3741                return ret;
3742
3743        if (qm->vfs_num) {
3744                ret = qm_set_vf_mse(qm, false);
3745                if (ret) {
3746                        pci_err(pdev, "Fails to disable vf MSE bit.\n");
3747                        return ret;
3748                }
3749        }
3750
3751        ret = qm_set_msi(qm, false);
3752        if (ret) {
3753                pci_err(pdev, "Fails to disable PEH MSI bit.\n");
3754                return ret;
3755        }
3756
3757        qm_dev_ecc_mbit_handle(qm);
3758
3759        /* OOO register set and check */
3760        writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
3761               qm->io_base + ACC_MASTER_GLOBAL_CTRL);
3762
3763        /* If bus lock, reset chip */
3764        ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
3765                                         val,
3766                                         (val == ACC_MASTER_TRANS_RETURN_RW),
3767                                         POLL_PERIOD, POLL_TIMEOUT);
3768        if (ret) {
3769                pci_emerg(pdev, "Bus lock! Please reset system.\n");
3770                return ret;
3771        }
3772
3773        ret = qm_set_pf_mse(qm, false);
3774        if (ret) {
3775                pci_err(pdev, "Fails to disable pf MSE bit.\n");
3776                return ret;
3777        }
3778
3779        /* The reset related sub-control registers are not in PCI BAR */
3780        if (ACPI_HANDLE(&pdev->dev)) {
3781                unsigned long long value = 0;
3782                acpi_status s;
3783
3784                s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
3785                                          qm->err_info.acpi_rst,
3786                                          NULL, &value);
3787                if (ACPI_FAILURE(s)) {
3788                        pci_err(pdev, "NO controller reset method!\n");
3789                        return -EIO;
3790                }
3791
3792                if (value) {
3793                        pci_err(pdev, "Reset step %llu failed!\n", value);
3794                        return -EIO;
3795                }
3796        } else {
3797                pci_err(pdev, "No reset method!\n");
3798                return -EINVAL;
3799        }
3800
3801        return 0;
3802}
3803
3804static int qm_vf_reset_done(struct hisi_qm *qm)
3805{
3806        struct hisi_qm_list *qm_list = qm->qm_list;
3807        struct pci_dev *pdev = qm->pdev;
3808        struct pci_dev *virtfn;
3809        struct hisi_qm *vf_qm;
3810        int ret = 0;
3811
3812        mutex_lock(&qm_list->lock);
3813        list_for_each_entry(vf_qm, &qm_list->list, list) {
3814                virtfn = vf_qm->pdev;
3815                if (virtfn == pdev)
3816                        continue;
3817
3818                if (pci_physfn(virtfn) == pdev) {
3819                        /* enable VFs PCIE BAR configuration */
3820                        pci_restore_state(virtfn);
3821
3822                        ret = qm_restart(vf_qm);
3823                        if (ret)
3824                                goto restart_fail;
3825                }
3826        }
3827
3828restart_fail:
3829        mutex_unlock(&qm_list->lock);
3830        return ret;
3831}
3832
3833static u32 qm_get_dev_err_status(struct hisi_qm *qm)
3834{
3835        return qm->err_ini->get_dev_hw_err_status(qm);
3836}
3837
3838static int qm_dev_hw_init(struct hisi_qm *qm)
3839{
3840        return qm->err_ini->hw_init(qm);
3841}
3842
3843static void qm_restart_prepare(struct hisi_qm *qm)
3844{
3845        u32 value;
3846
3847        if (!qm->err_status.is_qm_ecc_mbit &&
3848            !qm->err_status.is_dev_ecc_mbit)
3849                return;
3850
3851        /* temporarily close the OOO port used for PEH to write out MSI */
3852        value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3853        writel(value & ~qm->err_info.msi_wr_port,
3854               qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3855
3856        /* clear dev ecc 2bit error source if having */
3857        value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
3858        if (value && qm->err_ini->clear_dev_hw_err_status)
3859                qm->err_ini->clear_dev_hw_err_status(qm, value);
3860
3861        /* clear QM ecc mbit error source */
3862        writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
3863
3864        /* clear AM Reorder Buffer ecc mbit source */
3865        writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
3866
3867        if (qm->err_ini->open_axi_master_ooo)
3868                qm->err_ini->open_axi_master_ooo(qm);
3869}
3870
3871static void qm_restart_done(struct hisi_qm *qm)
3872{
3873        u32 value;
3874
3875        if (!qm->err_status.is_qm_ecc_mbit &&
3876            !qm->err_status.is_dev_ecc_mbit)
3877                return;
3878
3879        /* open the OOO port for PEH to write out MSI */
3880        value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3881        value |= qm->err_info.msi_wr_port;
3882        writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
3883
3884        qm->err_status.is_qm_ecc_mbit = false;
3885        qm->err_status.is_dev_ecc_mbit = false;
3886}
3887
3888static int qm_controller_reset_done(struct hisi_qm *qm)
3889{
3890        struct pci_dev *pdev = qm->pdev;
3891        int ret;
3892
3893        ret = qm_set_msi(qm, true);
3894        if (ret) {
3895                pci_err(pdev, "Fails to enable PEH MSI bit!\n");
3896                return ret;
3897        }
3898
3899        ret = qm_set_pf_mse(qm, true);
3900        if (ret) {
3901                pci_err(pdev, "Fails to enable pf MSE bit!\n");
3902                return ret;
3903        }
3904
3905        if (qm->vfs_num) {
3906                ret = qm_set_vf_mse(qm, true);
3907                if (ret) {
3908                        pci_err(pdev, "Fails to enable vf MSE bit!\n");
3909                        return ret;
3910                }
3911        }
3912
3913        ret = qm_dev_hw_init(qm);
3914        if (ret) {
3915                pci_err(pdev, "Failed to init device\n");
3916                return ret;
3917        }
3918
3919        qm_restart_prepare(qm);
3920
3921        ret = qm_restart(qm);
3922        if (ret) {
3923                pci_err(pdev, "Failed to start QM!\n");
3924                return ret;
3925        }
3926
3927        if (qm->vfs_num) {
3928                ret = qm_vf_q_assign(qm, qm->vfs_num);
3929                if (ret) {
3930                        pci_err(pdev, "Failed to assign queue!\n");
3931                        return ret;
3932                }
3933        }
3934
3935        ret = qm_vf_reset_done(qm);
3936        if (ret) {
3937                pci_err(pdev, "Failed to start VFs!\n");
3938                return -EPERM;
3939        }
3940
3941        hisi_qm_dev_err_init(qm);
3942        qm_restart_done(qm);
3943
3944        clear_bit(QM_RESETTING, &qm->misc_ctl);
3945
3946        return 0;
3947}
3948
3949static int qm_controller_reset(struct hisi_qm *qm)
3950{
3951        struct pci_dev *pdev = qm->pdev;
3952        int ret;
3953
3954        pci_info(pdev, "Controller resetting...\n");
3955
3956        ret = qm_controller_reset_prepare(qm);
3957        if (ret) {
3958                clear_bit(QM_RST_SCHED, &qm->misc_ctl);
3959                return ret;
3960        }
3961
3962        ret = qm_soft_reset(qm);
3963        if (ret) {
3964                pci_err(pdev, "Controller reset failed (%d)\n", ret);
3965                clear_bit(QM_RESETTING, &qm->misc_ctl);
3966                return ret;
3967        }
3968
3969        ret = qm_controller_reset_done(qm);
3970        if (ret) {
3971                clear_bit(QM_RESETTING, &qm->misc_ctl);
3972                return ret;
3973        }
3974
3975        pci_info(pdev, "Controller reset complete\n");
3976
3977        return 0;
3978}
3979
3980/**
3981 * hisi_qm_dev_slot_reset() - slot reset
3982 * @pdev: the PCIe device
3983 *
3984 * This function offers QM relate PCIe device reset interface. Drivers which
3985 * use QM can use this function as slot_reset in its struct pci_error_handlers.
3986 */
3987pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
3988{
3989        struct hisi_qm *qm = pci_get_drvdata(pdev);
3990        int ret;
3991
3992        if (pdev->is_virtfn)
3993                return PCI_ERS_RESULT_RECOVERED;
3994
3995        pci_aer_clear_nonfatal_status(pdev);
3996
3997        /* reset pcie device controller */
3998        ret = qm_controller_reset(qm);
3999        if (ret) {
4000                pci_err(pdev, "Controller reset failed (%d)\n", ret);
4001                return PCI_ERS_RESULT_DISCONNECT;
4002        }
4003
4004        return PCI_ERS_RESULT_RECOVERED;
4005}
4006EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
4007
4008/* check the interrupt is ecc-mbit error or not */
4009static int qm_check_dev_error(struct hisi_qm *qm)
4010{
4011        int ret;
4012
4013        if (qm->fun_type == QM_HW_VF)
4014                return 0;
4015
4016        ret = qm_get_hw_error_status(qm) & QM_ECC_MBIT;
4017        if (ret)
4018                return ret;
4019
4020        return (qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask);
4021}
4022
4023void hisi_qm_reset_prepare(struct pci_dev *pdev)
4024{
4025        struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4026        struct hisi_qm *qm = pci_get_drvdata(pdev);
4027        u32 delay = 0;
4028        int ret;
4029
4030        hisi_qm_dev_err_uninit(pf_qm);
4031
4032        /*
4033         * Check whether there is an ECC mbit error, If it occurs, need to
4034         * wait for soft reset to fix it.
4035         */
4036        while (qm_check_dev_error(pf_qm)) {
4037                msleep(++delay);
4038                if (delay > QM_RESET_WAIT_TIMEOUT)
4039                        return;
4040        }
4041
4042        ret = qm_reset_prepare_ready(qm);
4043        if (ret) {
4044                pci_err(pdev, "FLR not ready!\n");
4045                return;
4046        }
4047
4048        if (qm->vfs_num) {
4049                ret = qm_vf_reset_prepare(qm, QM_FLR);
4050                if (ret) {
4051                        pci_err(pdev, "Failed to prepare reset, ret = %d.\n",
4052                                ret);
4053                        return;
4054                }
4055        }
4056
4057        ret = hisi_qm_stop(qm, QM_FLR);
4058        if (ret) {
4059                pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
4060                return;
4061        }
4062
4063        pci_info(pdev, "FLR resetting...\n");
4064}
4065EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
4066
4067static bool qm_flr_reset_complete(struct pci_dev *pdev)
4068{
4069        struct pci_dev *pf_pdev = pci_physfn(pdev);
4070        struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
4071        u32 id;
4072
4073        pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
4074        if (id == QM_PCI_COMMAND_INVALID) {
4075                pci_err(pdev, "Device can not be used!\n");
4076                return false;
4077        }
4078
4079        return true;
4080}
4081
4082void hisi_qm_reset_done(struct pci_dev *pdev)
4083{
4084        struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4085        struct hisi_qm *qm = pci_get_drvdata(pdev);
4086        int ret;
4087
4088        hisi_qm_dev_err_init(pf_qm);
4089
4090        ret = qm_restart(qm);
4091        if (ret) {
4092                pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
4093                goto flr_done;
4094        }
4095
4096        if (qm->fun_type == QM_HW_PF) {
4097                ret = qm_dev_hw_init(qm);
4098                if (ret) {
4099                        pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
4100                        goto flr_done;
4101                }
4102
4103                if (!qm->vfs_num)
4104                        goto flr_done;
4105
4106                ret = qm_vf_q_assign(qm, qm->vfs_num);
4107                if (ret) {
4108                        pci_err(pdev, "Failed to assign VFs, ret = %d.\n", ret);
4109                        goto flr_done;
4110                }
4111
4112                ret = qm_vf_reset_done(qm);
4113                if (ret) {
4114                        pci_err(pdev, "Failed to start VFs, ret = %d.\n", ret);
4115                        goto flr_done;
4116                }
4117        }
4118
4119flr_done:
4120        if (qm_flr_reset_complete(pdev))
4121                pci_info(pdev, "FLR reset complete\n");
4122
4123        clear_bit(QM_RESETTING, &qm->misc_ctl);
4124}
4125EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
4126
4127static irqreturn_t qm_abnormal_irq(int irq, void *data)
4128{
4129        struct hisi_qm *qm = data;
4130        enum acc_err_result ret;
4131
4132        atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
4133        ret = qm_process_dev_error(qm);
4134        if (ret == ACC_ERR_NEED_RESET &&
4135            !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&
4136            !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))
4137                schedule_work(&qm->rst_work);
4138
4139        return IRQ_HANDLED;
4140}
4141
4142static int qm_irq_register(struct hisi_qm *qm)
4143{
4144        struct pci_dev *pdev = qm->pdev;
4145        int ret;
4146
4147        ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR),
4148                          qm_irq, 0, qm->dev_name, qm);
4149        if (ret)
4150                return ret;
4151
4152        if (qm->ver != QM_HW_V1) {
4153                ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
4154                                  qm_aeq_irq, 0, qm->dev_name, qm);
4155                if (ret)
4156                        goto err_aeq_irq;
4157
4158                if (qm->fun_type == QM_HW_PF) {
4159                        ret = request_irq(pci_irq_vector(pdev,
4160                                          QM_ABNORMAL_EVENT_IRQ_VECTOR),
4161                                          qm_abnormal_irq, 0, qm->dev_name, qm);
4162                        if (ret)
4163                                goto err_abonormal_irq;
4164                }
4165        }
4166
4167        return 0;
4168
4169err_abonormal_irq:
4170        free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
4171err_aeq_irq:
4172        free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
4173        return ret;
4174}
4175
4176/**
4177 * hisi_qm_dev_shutdown() - Shutdown device.
4178 * @pdev: The device will be shutdown.
4179 *
4180 * This function will stop qm when OS shutdown or rebooting.
4181 */
4182void hisi_qm_dev_shutdown(struct pci_dev *pdev)
4183{
4184        struct hisi_qm *qm = pci_get_drvdata(pdev);
4185        int ret;
4186
4187        ret = hisi_qm_stop(qm, QM_NORMAL);
4188        if (ret)
4189                dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
4190}
4191EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
4192
4193static void hisi_qm_controller_reset(struct work_struct *rst_work)
4194{
4195        struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
4196        int ret;
4197
4198        /* reset pcie device controller */
4199        ret = qm_controller_reset(qm);
4200        if (ret)
4201                dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
4202
4203}
4204
4205/**
4206 * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list.
4207 * @qm: The qm needs add.
4208 * @qm_list: The qm list.
4209 *
4210 * This function adds qm to qm list, and will register algorithm to
4211 * crypto when the qm list is empty.
4212 */
4213int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
4214{
4215        int flag = 0;
4216        int ret = 0;
4217