linux/drivers/misc/habanalabs/goya/goya.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2
   3/*
   4 * Copyright 2016-2019 HabanaLabs, Ltd.
   5 * All Rights Reserved.
   6 */
   7
   8#include "goyaP.h"
   9#include "../include/hw_ip/mmu/mmu_general.h"
  10#include "../include/hw_ip/mmu/mmu_v1_0.h"
  11#include "../include/goya/asic_reg/goya_masks.h"
  12#include "../include/goya/goya_reg_map.h"
  13
  14#include <linux/pci.h>
  15#include <linux/hwmon.h>
  16#include <linux/iommu.h>
  17#include <linux/seq_file.h>
  18
  19/*
  20 * GOYA security scheme:
  21 *
  22 * 1. Host is protected by:
  23 *        - Range registers (When MMU is enabled, DMA RR does NOT protect host)
  24 *        - MMU
  25 *
  26 * 2. DRAM is protected by:
  27 *        - Range registers (protect the first 512MB)
  28 *        - MMU (isolation between users)
  29 *
  30 * 3. Configuration is protected by:
  31 *        - Range registers
  32 *        - Protection bits
  33 *
  34 * When MMU is disabled:
  35 *
  36 * QMAN DMA: PQ, CQ, CP, DMA are secured.
  37 * PQ, CB and the data are on the host.
  38 *
  39 * QMAN TPC/MME:
  40 * PQ, CQ and CP are not secured.
  41 * PQ, CB and the data are on the SRAM/DRAM.
  42 *
  43 * Since QMAN DMA is secured, the driver is parsing the DMA CB:
  44 *     - checks DMA pointer
  45 *     - WREG, MSG_PROT are not allowed.
  46 *     - MSG_LONG/SHORT are allowed.
  47 *
  48 * A read/write transaction by the QMAN to a protected area will succeed if
  49 * and only if the QMAN's CP is secured and MSG_PROT is used
  50 *
  51 *
  52 * When MMU is enabled:
  53 *
  54 * QMAN DMA: PQ, CQ and CP are secured.
  55 * MMU is set to bypass on the Secure props register of the QMAN.
  56 * The reasons we don't enable MMU for PQ, CQ and CP are:
  57 *     - PQ entry is in kernel address space and the driver doesn't map it.
  58 *     - CP writes to MSIX register and to kernel address space (completion
  59 *       queue).
  60 *
  61 * DMA is not secured but because CP is secured, the driver still needs to parse
  62 * the CB, but doesn't need to check the DMA addresses.
  63 *
  64 * For QMAN DMA 0, DMA is also secured because only the driver uses this DMA and
  65 * the driver doesn't map memory in MMU.
  66 *
  67 * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode)
  68 *
  69 * DMA RR does NOT protect host because DMA is not secured
  70 *
  71 */
  72
  73#define GOYA_BOOT_FIT_FILE      "habanalabs/goya/goya-boot-fit.itb"
  74#define GOYA_LINUX_FW_FILE      "habanalabs/goya/goya-fit.itb"
  75
  76#define GOYA_MMU_REGS_NUM               63
  77
  78#define GOYA_DMA_POOL_BLK_SIZE          0x100           /* 256 bytes */
  79
  80#define GOYA_RESET_TIMEOUT_MSEC         500             /* 500ms */
  81#define GOYA_PLDM_RESET_TIMEOUT_MSEC    20000           /* 20s */
  82#define GOYA_RESET_WAIT_MSEC            1               /* 1ms */
  83#define GOYA_CPU_RESET_WAIT_MSEC        100             /* 100ms */
  84#define GOYA_PLDM_RESET_WAIT_MSEC       1000            /* 1s */
  85#define GOYA_TEST_QUEUE_WAIT_USEC       100000          /* 100ms */
  86#define GOYA_PLDM_MMU_TIMEOUT_USEC      (MMU_CONFIG_TIMEOUT_USEC * 100)
  87#define GOYA_PLDM_QMAN0_TIMEOUT_USEC    (HL_DEVICE_TIMEOUT_USEC * 30)
  88#define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC  1000000         /* 1s */
  89#define GOYA_MSG_TO_CPU_TIMEOUT_USEC    4000000         /* 4s */
  90#define GOYA_WAIT_FOR_BL_TIMEOUT_USEC   15000000        /* 15s */
  91
  92#define GOYA_QMAN0_FENCE_VAL            0xD169B243
  93
  94#define GOYA_MAX_STRING_LEN             20
  95
  96#define GOYA_CB_POOL_CB_CNT             512
  97#define GOYA_CB_POOL_CB_SIZE            0x20000         /* 128KB */
  98
  99#define IS_QM_IDLE(engine, qm_glbl_sts0) \
 100        (((qm_glbl_sts0) & engine##_QM_IDLE_MASK) == engine##_QM_IDLE_MASK)
 101#define IS_DMA_QM_IDLE(qm_glbl_sts0)    IS_QM_IDLE(DMA, qm_glbl_sts0)
 102#define IS_TPC_QM_IDLE(qm_glbl_sts0)    IS_QM_IDLE(TPC, qm_glbl_sts0)
 103#define IS_MME_QM_IDLE(qm_glbl_sts0)    IS_QM_IDLE(MME, qm_glbl_sts0)
 104
 105#define IS_CMDQ_IDLE(engine, cmdq_glbl_sts0) \
 106        (((cmdq_glbl_sts0) & engine##_CMDQ_IDLE_MASK) == \
 107                        engine##_CMDQ_IDLE_MASK)
 108#define IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) \
 109        IS_CMDQ_IDLE(TPC, cmdq_glbl_sts0)
 110#define IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) \
 111        IS_CMDQ_IDLE(MME, cmdq_glbl_sts0)
 112
 113#define IS_DMA_IDLE(dma_core_sts0) \
 114        !((dma_core_sts0) & DMA_CH_0_STS0_DMA_BUSY_MASK)
 115
 116#define IS_TPC_IDLE(tpc_cfg_sts) \
 117        (((tpc_cfg_sts) & TPC_CFG_IDLE_MASK) == TPC_CFG_IDLE_MASK)
 118
 119#define IS_MME_IDLE(mme_arch_sts) \
 120        (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
 121
 122static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = {
 123                "goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3",
 124                "goya cq 4", "goya cpu eq"
 125};
 126
 127static u16 goya_packet_sizes[MAX_PACKET_ID] = {
 128        [PACKET_WREG_32]        = sizeof(struct packet_wreg32),
 129        [PACKET_WREG_BULK]      = sizeof(struct packet_wreg_bulk),
 130        [PACKET_MSG_LONG]       = sizeof(struct packet_msg_long),
 131        [PACKET_MSG_SHORT]      = sizeof(struct packet_msg_short),
 132        [PACKET_CP_DMA]         = sizeof(struct packet_cp_dma),
 133        [PACKET_MSG_PROT]       = sizeof(struct packet_msg_prot),
 134        [PACKET_FENCE]          = sizeof(struct packet_fence),
 135        [PACKET_LIN_DMA]        = sizeof(struct packet_lin_dma),
 136        [PACKET_NOP]            = sizeof(struct packet_nop),
 137        [PACKET_STOP]           = sizeof(struct packet_stop)
 138};
 139
 140static inline bool validate_packet_id(enum packet_id id)
 141{
 142        switch (id) {
 143        case PACKET_WREG_32:
 144        case PACKET_WREG_BULK:
 145        case PACKET_MSG_LONG:
 146        case PACKET_MSG_SHORT:
 147        case PACKET_CP_DMA:
 148        case PACKET_MSG_PROT:
 149        case PACKET_FENCE:
 150        case PACKET_LIN_DMA:
 151        case PACKET_NOP:
 152        case PACKET_STOP:
 153                return true;
 154        default:
 155                return false;
 156        }
 157}
 158
 159static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = {
 160        mmDMA_QM_0_GLBL_NON_SECURE_PROPS,
 161        mmDMA_QM_1_GLBL_NON_SECURE_PROPS,
 162        mmDMA_QM_2_GLBL_NON_SECURE_PROPS,
 163        mmDMA_QM_3_GLBL_NON_SECURE_PROPS,
 164        mmDMA_QM_4_GLBL_NON_SECURE_PROPS,
 165        mmTPC0_QM_GLBL_SECURE_PROPS,
 166        mmTPC0_QM_GLBL_NON_SECURE_PROPS,
 167        mmTPC0_CMDQ_GLBL_SECURE_PROPS,
 168        mmTPC0_CMDQ_GLBL_NON_SECURE_PROPS,
 169        mmTPC0_CFG_ARUSER,
 170        mmTPC0_CFG_AWUSER,
 171        mmTPC1_QM_GLBL_SECURE_PROPS,
 172        mmTPC1_QM_GLBL_NON_SECURE_PROPS,
 173        mmTPC1_CMDQ_GLBL_SECURE_PROPS,
 174        mmTPC1_CMDQ_GLBL_NON_SECURE_PROPS,
 175        mmTPC1_CFG_ARUSER,
 176        mmTPC1_CFG_AWUSER,
 177        mmTPC2_QM_GLBL_SECURE_PROPS,
 178        mmTPC2_QM_GLBL_NON_SECURE_PROPS,
 179        mmTPC2_CMDQ_GLBL_SECURE_PROPS,
 180        mmTPC2_CMDQ_GLBL_NON_SECURE_PROPS,
 181        mmTPC2_CFG_ARUSER,
 182        mmTPC2_CFG_AWUSER,
 183        mmTPC3_QM_GLBL_SECURE_PROPS,
 184        mmTPC3_QM_GLBL_NON_SECURE_PROPS,
 185        mmTPC3_CMDQ_GLBL_SECURE_PROPS,
 186        mmTPC3_CMDQ_GLBL_NON_SECURE_PROPS,
 187        mmTPC3_CFG_ARUSER,
 188        mmTPC3_CFG_AWUSER,
 189        mmTPC4_QM_GLBL_SECURE_PROPS,
 190        mmTPC4_QM_GLBL_NON_SECURE_PROPS,
 191        mmTPC4_CMDQ_GLBL_SECURE_PROPS,
 192        mmTPC4_CMDQ_GLBL_NON_SECURE_PROPS,
 193        mmTPC4_CFG_ARUSER,
 194        mmTPC4_CFG_AWUSER,
 195        mmTPC5_QM_GLBL_SECURE_PROPS,
 196        mmTPC5_QM_GLBL_NON_SECURE_PROPS,
 197        mmTPC5_CMDQ_GLBL_SECURE_PROPS,
 198        mmTPC5_CMDQ_GLBL_NON_SECURE_PROPS,
 199        mmTPC5_CFG_ARUSER,
 200        mmTPC5_CFG_AWUSER,
 201        mmTPC6_QM_GLBL_SECURE_PROPS,
 202        mmTPC6_QM_GLBL_NON_SECURE_PROPS,
 203        mmTPC6_CMDQ_GLBL_SECURE_PROPS,
 204        mmTPC6_CMDQ_GLBL_NON_SECURE_PROPS,
 205        mmTPC6_CFG_ARUSER,
 206        mmTPC6_CFG_AWUSER,
 207        mmTPC7_QM_GLBL_SECURE_PROPS,
 208        mmTPC7_QM_GLBL_NON_SECURE_PROPS,
 209        mmTPC7_CMDQ_GLBL_SECURE_PROPS,
 210        mmTPC7_CMDQ_GLBL_NON_SECURE_PROPS,
 211        mmTPC7_CFG_ARUSER,
 212        mmTPC7_CFG_AWUSER,
 213        mmMME_QM_GLBL_SECURE_PROPS,
 214        mmMME_QM_GLBL_NON_SECURE_PROPS,
 215        mmMME_CMDQ_GLBL_SECURE_PROPS,
 216        mmMME_CMDQ_GLBL_NON_SECURE_PROPS,
 217        mmMME_SBA_CONTROL_DATA,
 218        mmMME_SBB_CONTROL_DATA,
 219        mmMME_SBC_CONTROL_DATA,
 220        mmMME_WBC_CONTROL_DATA,
 221        mmPCIE_WRAP_PSOC_ARUSER,
 222        mmPCIE_WRAP_PSOC_AWUSER
 223};
 224
 225static u32 goya_all_events[] = {
 226        GOYA_ASYNC_EVENT_ID_PCIE_IF,
 227        GOYA_ASYNC_EVENT_ID_TPC0_ECC,
 228        GOYA_ASYNC_EVENT_ID_TPC1_ECC,
 229        GOYA_ASYNC_EVENT_ID_TPC2_ECC,
 230        GOYA_ASYNC_EVENT_ID_TPC3_ECC,
 231        GOYA_ASYNC_EVENT_ID_TPC4_ECC,
 232        GOYA_ASYNC_EVENT_ID_TPC5_ECC,
 233        GOYA_ASYNC_EVENT_ID_TPC6_ECC,
 234        GOYA_ASYNC_EVENT_ID_TPC7_ECC,
 235        GOYA_ASYNC_EVENT_ID_MME_ECC,
 236        GOYA_ASYNC_EVENT_ID_MME_ECC_EXT,
 237        GOYA_ASYNC_EVENT_ID_MMU_ECC,
 238        GOYA_ASYNC_EVENT_ID_DMA_MACRO,
 239        GOYA_ASYNC_EVENT_ID_DMA_ECC,
 240        GOYA_ASYNC_EVENT_ID_CPU_IF_ECC,
 241        GOYA_ASYNC_EVENT_ID_PSOC_MEM,
 242        GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT,
 243        GOYA_ASYNC_EVENT_ID_SRAM0,
 244        GOYA_ASYNC_EVENT_ID_SRAM1,
 245        GOYA_ASYNC_EVENT_ID_SRAM2,
 246        GOYA_ASYNC_EVENT_ID_SRAM3,
 247        GOYA_ASYNC_EVENT_ID_SRAM4,
 248        GOYA_ASYNC_EVENT_ID_SRAM5,
 249        GOYA_ASYNC_EVENT_ID_SRAM6,
 250        GOYA_ASYNC_EVENT_ID_SRAM7,
 251        GOYA_ASYNC_EVENT_ID_SRAM8,
 252        GOYA_ASYNC_EVENT_ID_SRAM9,
 253        GOYA_ASYNC_EVENT_ID_SRAM10,
 254        GOYA_ASYNC_EVENT_ID_SRAM11,
 255        GOYA_ASYNC_EVENT_ID_SRAM12,
 256        GOYA_ASYNC_EVENT_ID_SRAM13,
 257        GOYA_ASYNC_EVENT_ID_SRAM14,
 258        GOYA_ASYNC_EVENT_ID_SRAM15,
 259        GOYA_ASYNC_EVENT_ID_SRAM16,
 260        GOYA_ASYNC_EVENT_ID_SRAM17,
 261        GOYA_ASYNC_EVENT_ID_SRAM18,
 262        GOYA_ASYNC_EVENT_ID_SRAM19,
 263        GOYA_ASYNC_EVENT_ID_SRAM20,
 264        GOYA_ASYNC_EVENT_ID_SRAM21,
 265        GOYA_ASYNC_EVENT_ID_SRAM22,
 266        GOYA_ASYNC_EVENT_ID_SRAM23,
 267        GOYA_ASYNC_EVENT_ID_SRAM24,
 268        GOYA_ASYNC_EVENT_ID_SRAM25,
 269        GOYA_ASYNC_EVENT_ID_SRAM26,
 270        GOYA_ASYNC_EVENT_ID_SRAM27,
 271        GOYA_ASYNC_EVENT_ID_SRAM28,
 272        GOYA_ASYNC_EVENT_ID_SRAM29,
 273        GOYA_ASYNC_EVENT_ID_GIC500,
 274        GOYA_ASYNC_EVENT_ID_PLL0,
 275        GOYA_ASYNC_EVENT_ID_PLL1,
 276        GOYA_ASYNC_EVENT_ID_PLL3,
 277        GOYA_ASYNC_EVENT_ID_PLL4,
 278        GOYA_ASYNC_EVENT_ID_PLL5,
 279        GOYA_ASYNC_EVENT_ID_PLL6,
 280        GOYA_ASYNC_EVENT_ID_AXI_ECC,
 281        GOYA_ASYNC_EVENT_ID_L2_RAM_ECC,
 282        GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET,
 283        GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT,
 284        GOYA_ASYNC_EVENT_ID_PCIE_DEC,
 285        GOYA_ASYNC_EVENT_ID_TPC0_DEC,
 286        GOYA_ASYNC_EVENT_ID_TPC1_DEC,
 287        GOYA_ASYNC_EVENT_ID_TPC2_DEC,
 288        GOYA_ASYNC_EVENT_ID_TPC3_DEC,
 289        GOYA_ASYNC_EVENT_ID_TPC4_DEC,
 290        GOYA_ASYNC_EVENT_ID_TPC5_DEC,
 291        GOYA_ASYNC_EVENT_ID_TPC6_DEC,
 292        GOYA_ASYNC_EVENT_ID_TPC7_DEC,
 293        GOYA_ASYNC_EVENT_ID_MME_WACS,
 294        GOYA_ASYNC_EVENT_ID_MME_WACSD,
 295        GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER,
 296        GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC,
 297        GOYA_ASYNC_EVENT_ID_PSOC,
 298        GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR,
 299        GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR,
 300        GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR,
 301        GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR,
 302        GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR,
 303        GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR,
 304        GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR,
 305        GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR,
 306        GOYA_ASYNC_EVENT_ID_TPC0_CMDQ,
 307        GOYA_ASYNC_EVENT_ID_TPC1_CMDQ,
 308        GOYA_ASYNC_EVENT_ID_TPC2_CMDQ,
 309        GOYA_ASYNC_EVENT_ID_TPC3_CMDQ,
 310        GOYA_ASYNC_EVENT_ID_TPC4_CMDQ,
 311        GOYA_ASYNC_EVENT_ID_TPC5_CMDQ,
 312        GOYA_ASYNC_EVENT_ID_TPC6_CMDQ,
 313        GOYA_ASYNC_EVENT_ID_TPC7_CMDQ,
 314        GOYA_ASYNC_EVENT_ID_TPC0_QM,
 315        GOYA_ASYNC_EVENT_ID_TPC1_QM,
 316        GOYA_ASYNC_EVENT_ID_TPC2_QM,
 317        GOYA_ASYNC_EVENT_ID_TPC3_QM,
 318        GOYA_ASYNC_EVENT_ID_TPC4_QM,
 319        GOYA_ASYNC_EVENT_ID_TPC5_QM,
 320        GOYA_ASYNC_EVENT_ID_TPC6_QM,
 321        GOYA_ASYNC_EVENT_ID_TPC7_QM,
 322        GOYA_ASYNC_EVENT_ID_MME_QM,
 323        GOYA_ASYNC_EVENT_ID_MME_CMDQ,
 324        GOYA_ASYNC_EVENT_ID_DMA0_QM,
 325        GOYA_ASYNC_EVENT_ID_DMA1_QM,
 326        GOYA_ASYNC_EVENT_ID_DMA2_QM,
 327        GOYA_ASYNC_EVENT_ID_DMA3_QM,
 328        GOYA_ASYNC_EVENT_ID_DMA4_QM,
 329        GOYA_ASYNC_EVENT_ID_DMA0_CH,
 330        GOYA_ASYNC_EVENT_ID_DMA1_CH,
 331        GOYA_ASYNC_EVENT_ID_DMA2_CH,
 332        GOYA_ASYNC_EVENT_ID_DMA3_CH,
 333        GOYA_ASYNC_EVENT_ID_DMA4_CH,
 334        GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU,
 335        GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU,
 336        GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU,
 337        GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU,
 338        GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU,
 339        GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU,
 340        GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU,
 341        GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU,
 342        GOYA_ASYNC_EVENT_ID_DMA_BM_CH0,
 343        GOYA_ASYNC_EVENT_ID_DMA_BM_CH1,
 344        GOYA_ASYNC_EVENT_ID_DMA_BM_CH2,
 345        GOYA_ASYNC_EVENT_ID_DMA_BM_CH3,
 346        GOYA_ASYNC_EVENT_ID_DMA_BM_CH4,
 347        GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S,
 348        GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E,
 349        GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S,
 350        GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E
 351};
 352
 353static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
 354static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
 355static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
 356static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
 357
 358int goya_set_fixed_properties(struct hl_device *hdev)
 359{
 360        struct asic_fixed_properties *prop = &hdev->asic_prop;
 361        int i;
 362
 363        prop->max_queues = GOYA_QUEUE_ID_SIZE;
 364        prop->hw_queues_props = kcalloc(prop->max_queues,
 365                        sizeof(struct hw_queue_properties),
 366                        GFP_KERNEL);
 367
 368        if (!prop->hw_queues_props)
 369                return -ENOMEM;
 370
 371        for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
 372                prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
 373                prop->hw_queues_props[i].driver_only = 0;
 374                prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL;
 375        }
 376
 377        for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) {
 378                prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
 379                prop->hw_queues_props[i].driver_only = 1;
 380                prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL;
 381        }
 382
 383        for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES +
 384                        NUMBER_OF_INT_HW_QUEUES; i++) {
 385                prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
 386                prop->hw_queues_props[i].driver_only = 0;
 387                prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER;
 388        }
 389
 390        prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
 391
 392        prop->dram_base_address = DRAM_PHYS_BASE;
 393        prop->dram_size = DRAM_PHYS_DEFAULT_SIZE;
 394        prop->dram_end_address = prop->dram_base_address + prop->dram_size;
 395        prop->dram_user_base_address = DRAM_BASE_ADDR_USER;
 396
 397        prop->sram_base_address = SRAM_BASE_ADDR;
 398        prop->sram_size = SRAM_SIZE;
 399        prop->sram_end_address = prop->sram_base_address + prop->sram_size;
 400        prop->sram_user_base_address = prop->sram_base_address +
 401                                                SRAM_USER_BASE_OFFSET;
 402
 403        prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
 404        prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR;
 405        if (hdev->pldm)
 406                prop->mmu_pgt_size = 0x800000; /* 8MB */
 407        else
 408                prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
 409        prop->mmu_pte_size = HL_PTE_SIZE;
 410        prop->mmu_hop_table_size = HOP_TABLE_SIZE;
 411        prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
 412        prop->dram_page_size = PAGE_SIZE_2MB;
 413        prop->dram_supports_virtual_memory = true;
 414
 415        prop->dmmu.hop0_shift = HOP0_SHIFT;
 416        prop->dmmu.hop1_shift = HOP1_SHIFT;
 417        prop->dmmu.hop2_shift = HOP2_SHIFT;
 418        prop->dmmu.hop3_shift = HOP3_SHIFT;
 419        prop->dmmu.hop4_shift = HOP4_SHIFT;
 420        prop->dmmu.hop0_mask = HOP0_MASK;
 421        prop->dmmu.hop1_mask = HOP1_MASK;
 422        prop->dmmu.hop2_mask = HOP2_MASK;
 423        prop->dmmu.hop3_mask = HOP3_MASK;
 424        prop->dmmu.hop4_mask = HOP4_MASK;
 425        prop->dmmu.start_addr = VA_DDR_SPACE_START;
 426        prop->dmmu.end_addr = VA_DDR_SPACE_END;
 427        prop->dmmu.page_size = PAGE_SIZE_2MB;
 428        prop->dmmu.num_hops = MMU_ARCH_5_HOPS;
 429
 430        /* shifts and masks are the same in PMMU and DMMU */
 431        memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu));
 432        prop->pmmu.start_addr = VA_HOST_SPACE_START;
 433        prop->pmmu.end_addr = VA_HOST_SPACE_END;
 434        prop->pmmu.page_size = PAGE_SIZE_4KB;
 435        prop->pmmu.num_hops = MMU_ARCH_5_HOPS;
 436
 437        /* PMMU and HPMMU are the same except of page size */
 438        memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
 439        prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
 440
 441        prop->dram_size_for_default_page_mapping = VA_DDR_SPACE_END;
 442        prop->cfg_size = CFG_SIZE;
 443        prop->max_asid = MAX_ASID;
 444        prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE;
 445        prop->high_pll = PLL_HIGH_DEFAULT;
 446        prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT;
 447        prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE;
 448        prop->max_power_default = MAX_POWER_DEFAULT;
 449        prop->dc_power_default = DC_POWER_DEFAULT;
 450        prop->tpc_enabled_mask = TPC_ENABLED_MASK;
 451        prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
 452        prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
 453
 454        strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
 455                CARD_NAME_MAX_LEN);
 456
 457        prop->max_pending_cs = GOYA_MAX_PENDING_CS;
 458
 459        prop->first_available_user_msix_interrupt = USHRT_MAX;
 460
 461        for (i = 0 ; i < HL_MAX_DCORES ; i++)
 462                prop->first_available_cq[i] = USHRT_MAX;
 463
 464        prop->fw_cpu_boot_dev_sts0_valid = false;
 465        prop->fw_cpu_boot_dev_sts1_valid = false;
 466        prop->hard_reset_done_by_fw = false;
 467        prop->gic_interrupts_enable = true;
 468
 469        return 0;
 470}
 471
 472/*
 473 * goya_pci_bars_map - Map PCI BARS of Goya device
 474 *
 475 * @hdev: pointer to hl_device structure
 476 *
 477 * Request PCI regions and map them to kernel virtual addresses.
 478 * Returns 0 on success
 479 *
 480 */
 481static int goya_pci_bars_map(struct hl_device *hdev)
 482{
 483        static const char * const name[] = {"SRAM_CFG", "MSIX", "DDR"};
 484        bool is_wc[3] = {false, false, true};
 485        int rc;
 486
 487        rc = hl_pci_bars_map(hdev, name, is_wc);
 488        if (rc)
 489                return rc;
 490
 491        hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] +
 492                        (CFG_BASE - SRAM_BASE_ADDR);
 493
 494        return 0;
 495}
 496
 497static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
 498{
 499        struct goya_device *goya = hdev->asic_specific;
 500        struct hl_inbound_pci_region pci_region;
 501        u64 old_addr = addr;
 502        int rc;
 503
 504        if ((goya) && (goya->ddr_bar_cur_addr == addr))
 505                return old_addr;
 506
 507        /* Inbound Region 1 - Bar 4 - Point to DDR */
 508        pci_region.mode = PCI_BAR_MATCH_MODE;
 509        pci_region.bar = DDR_BAR_ID;
 510        pci_region.addr = addr;
 511        rc = hl_pci_set_inbound_region(hdev, 1, &pci_region);
 512        if (rc)
 513                return U64_MAX;
 514
 515        if (goya) {
 516                old_addr = goya->ddr_bar_cur_addr;
 517                goya->ddr_bar_cur_addr = addr;
 518        }
 519
 520        return old_addr;
 521}
 522
 523/*
 524 * goya_init_iatu - Initialize the iATU unit inside the PCI controller
 525 *
 526 * @hdev: pointer to hl_device structure
 527 *
 528 * This is needed in case the firmware doesn't initialize the iATU
 529 *
 530 */
 531static int goya_init_iatu(struct hl_device *hdev)
 532{
 533        struct hl_inbound_pci_region inbound_region;
 534        struct hl_outbound_pci_region outbound_region;
 535        int rc;
 536
 537        if (hdev->asic_prop.iatu_done_by_fw)
 538                return 0;
 539
 540        /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */
 541        inbound_region.mode = PCI_BAR_MATCH_MODE;
 542        inbound_region.bar = SRAM_CFG_BAR_ID;
 543        inbound_region.addr = SRAM_BASE_ADDR;
 544        rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
 545        if (rc)
 546                goto done;
 547
 548        /* Inbound Region 1 - Bar 4 - Point to DDR */
 549        inbound_region.mode = PCI_BAR_MATCH_MODE;
 550        inbound_region.bar = DDR_BAR_ID;
 551        inbound_region.addr = DRAM_PHYS_BASE;
 552        rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
 553        if (rc)
 554                goto done;
 555
 556        hdev->asic_funcs->set_dma_mask_from_fw(hdev);
 557
 558        /* Outbound Region 0 - Point to Host  */
 559        outbound_region.addr = HOST_PHYS_BASE;
 560        outbound_region.size = HOST_PHYS_SIZE;
 561        rc = hl_pci_set_outbound_region(hdev, &outbound_region);
 562
 563done:
 564        return rc;
 565}
 566
 567static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
 568{
 569        return RREG32(mmHW_STATE);
 570}
 571
 572/*
 573 * goya_early_init - GOYA early initialization code
 574 *
 575 * @hdev: pointer to hl_device structure
 576 *
 577 * Verify PCI bars
 578 * Set DMA masks
 579 * PCI controller initialization
 580 * Map PCI bars
 581 *
 582 */
 583static int goya_early_init(struct hl_device *hdev)
 584{
 585        struct asic_fixed_properties *prop = &hdev->asic_prop;
 586        struct pci_dev *pdev = hdev->pdev;
 587        u32 fw_boot_status, val;
 588        int rc;
 589
 590        rc = goya_set_fixed_properties(hdev);
 591        if (rc) {
 592                dev_err(hdev->dev, "Failed to get fixed properties\n");
 593                return rc;
 594        }
 595
 596        /* Check BAR sizes */
 597        if (pci_resource_len(pdev, SRAM_CFG_BAR_ID) != CFG_BAR_SIZE) {
 598                dev_err(hdev->dev,
 599                        "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
 600                        SRAM_CFG_BAR_ID,
 601                        (unsigned long long) pci_resource_len(pdev,
 602                                                        SRAM_CFG_BAR_ID),
 603                        CFG_BAR_SIZE);
 604                rc = -ENODEV;
 605                goto free_queue_props;
 606        }
 607
 608        if (pci_resource_len(pdev, MSIX_BAR_ID) != MSIX_BAR_SIZE) {
 609                dev_err(hdev->dev,
 610                        "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
 611                        MSIX_BAR_ID,
 612                        (unsigned long long) pci_resource_len(pdev,
 613                                                                MSIX_BAR_ID),
 614                        MSIX_BAR_SIZE);
 615                rc = -ENODEV;
 616                goto free_queue_props;
 617        }
 618
 619        prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
 620
 621        /* If FW security is enabled at this point it means no access to ELBI */
 622        if (hdev->asic_prop.fw_security_enabled) {
 623                hdev->asic_prop.iatu_done_by_fw = true;
 624                goto pci_init;
 625        }
 626
 627        rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0,
 628                                &fw_boot_status);
 629        if (rc)
 630                goto free_queue_props;
 631
 632        /* Check whether FW is configuring iATU */
 633        if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) &&
 634                        (fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN))
 635                hdev->asic_prop.iatu_done_by_fw = true;
 636
 637pci_init:
 638        rc = hl_pci_init(hdev);
 639        if (rc)
 640                goto free_queue_props;
 641
 642        /* Before continuing in the initialization, we need to read the preboot
 643         * version to determine whether we run with a security-enabled firmware
 644         */
 645        rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
 646                                        mmCPU_BOOT_DEV_STS0,
 647                                        mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
 648                                        mmCPU_BOOT_ERR1,
 649                                        GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
 650        if (rc) {
 651                if (hdev->reset_on_preboot_fail)
 652                        hdev->asic_funcs->hw_fini(hdev, true);
 653                goto pci_fini;
 654        }
 655
 656        if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
 657                dev_info(hdev->dev,
 658                        "H/W state is dirty, must reset before initializing\n");
 659                hdev->asic_funcs->hw_fini(hdev, true);
 660        }
 661
 662        if (!hdev->pldm) {
 663                val = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
 664                if (val & PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_SRIOV_EN_MASK)
 665                        dev_warn(hdev->dev,
 666                                "PCI strap is not configured correctly, PCI bus errors may occur\n");
 667        }
 668
 669        return 0;
 670
 671pci_fini:
 672        hl_pci_fini(hdev);
 673free_queue_props:
 674        kfree(hdev->asic_prop.hw_queues_props);
 675        return rc;
 676}
 677
 678/*
 679 * goya_early_fini - GOYA early finalization code
 680 *
 681 * @hdev: pointer to hl_device structure
 682 *
 683 * Unmap PCI bars
 684 *
 685 */
 686static int goya_early_fini(struct hl_device *hdev)
 687{
 688        kfree(hdev->asic_prop.hw_queues_props);
 689        hl_pci_fini(hdev);
 690
 691        return 0;
 692}
 693
 694static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
 695{
 696        /* mask to zero the MMBP and ASID bits */
 697        WREG32_AND(reg, ~0x7FF);
 698        WREG32_OR(reg, asid);
 699}
 700
 701static void goya_qman0_set_security(struct hl_device *hdev, bool secure)
 702{
 703        struct goya_device *goya = hdev->asic_specific;
 704
 705        if (!(goya->hw_cap_initialized & HW_CAP_MMU))
 706                return;
 707
 708        if (secure)
 709                WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED);
 710        else
 711                WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED);
 712
 713        RREG32(mmDMA_QM_0_GLBL_PROT);
 714}
 715
 716/*
 717 * goya_fetch_psoc_frequency - Fetch PSOC frequency values
 718 *
 719 * @hdev: pointer to hl_device structure
 720 *
 721 */
 722static void goya_fetch_psoc_frequency(struct hl_device *hdev)
 723{
 724        struct asic_fixed_properties *prop = &hdev->asic_prop;
 725        u32 nr = 0, nf = 0, od = 0, div_fctr = 0, pll_clk, div_sel;
 726        u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq;
 727        int rc;
 728
 729        if (hdev->asic_prop.fw_security_enabled) {
 730                rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL,
 731                                pll_freq_arr);
 732
 733                if (rc)
 734                        return;
 735
 736                freq = pll_freq_arr[1];
 737        } else {
 738                div_fctr = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
 739                div_sel = RREG32(mmPSOC_PCI_PLL_DIV_SEL_1);
 740                nr = RREG32(mmPSOC_PCI_PLL_NR);
 741                nf = RREG32(mmPSOC_PCI_PLL_NF);
 742                od = RREG32(mmPSOC_PCI_PLL_OD);
 743
 744                if (div_sel == DIV_SEL_REF_CLK ||
 745                                div_sel == DIV_SEL_DIVIDED_REF) {
 746                        if (div_sel == DIV_SEL_REF_CLK)
 747                                freq = PLL_REF_CLK;
 748                        else
 749                                freq = PLL_REF_CLK / (div_fctr + 1);
 750                } else if (div_sel == DIV_SEL_PLL_CLK ||
 751                                div_sel == DIV_SEL_DIVIDED_PLL) {
 752                        pll_clk = PLL_REF_CLK * (nf + 1) /
 753                                        ((nr + 1) * (od + 1));
 754                        if (div_sel == DIV_SEL_PLL_CLK)
 755                                freq = pll_clk;
 756                        else
 757                                freq = pll_clk / (div_fctr + 1);
 758                } else {
 759                        dev_warn(hdev->dev,
 760                                "Received invalid div select value: %d",
 761                                div_sel);
 762                        freq = 0;
 763                }
 764        }
 765
 766        prop->psoc_timestamp_frequency = freq;
 767        prop->psoc_pci_pll_nr = nr;
 768        prop->psoc_pci_pll_nf = nf;
 769        prop->psoc_pci_pll_od = od;
 770        prop->psoc_pci_pll_div_factor = div_fctr;
 771}
 772
 773int goya_late_init(struct hl_device *hdev)
 774{
 775        struct asic_fixed_properties *prop = &hdev->asic_prop;
 776        int rc;
 777
 778        goya_fetch_psoc_frequency(hdev);
 779
 780        rc = goya_mmu_clear_pgt_range(hdev);
 781        if (rc) {
 782                dev_err(hdev->dev,
 783                        "Failed to clear MMU page tables range %d\n", rc);
 784                return rc;
 785        }
 786
 787        rc = goya_mmu_set_dram_default_page(hdev);
 788        if (rc) {
 789                dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc);
 790                return rc;
 791        }
 792
 793        rc = goya_mmu_add_mappings_for_device_cpu(hdev);
 794        if (rc)
 795                return rc;
 796
 797        rc = goya_init_cpu_queues(hdev);
 798        if (rc)
 799                return rc;
 800
 801        rc = goya_test_cpu_queue(hdev);
 802        if (rc)
 803                return rc;
 804
 805        rc = goya_cpucp_info_get(hdev);
 806        if (rc) {
 807                dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc);
 808                return rc;
 809        }
 810
 811        /* Now that we have the DRAM size in ASIC prop, we need to check
 812         * its size and configure the DMA_IF DDR wrap protection (which is in
 813         * the MMU block) accordingly. The value is the log2 of the DRAM size
 814         */
 815        WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size));
 816
 817        rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS);
 818        if (rc) {
 819                dev_err(hdev->dev,
 820                        "Failed to enable PCI access from CPU %d\n", rc);
 821                return rc;
 822        }
 823
 824        return 0;
 825}
 826
 827/*
 828 * goya_late_fini - GOYA late tear-down code
 829 *
 830 * @hdev: pointer to hl_device structure
 831 *
 832 * Free sensors allocated structures
 833 */
 834void goya_late_fini(struct hl_device *hdev)
 835{
 836        const struct hwmon_channel_info **channel_info_arr;
 837        int i = 0;
 838
 839        if (!hdev->hl_chip_info->info)
 840                return;
 841
 842        channel_info_arr = hdev->hl_chip_info->info;
 843
 844        while (channel_info_arr[i]) {
 845                kfree(channel_info_arr[i]->config);
 846                kfree(channel_info_arr[i]);
 847                i++;
 848        }
 849
 850        kfree(channel_info_arr);
 851
 852        hdev->hl_chip_info->info = NULL;
 853}
 854
 855static void goya_set_pci_memory_regions(struct hl_device *hdev)
 856{
 857        struct asic_fixed_properties *prop = &hdev->asic_prop;
 858        struct pci_mem_region *region;
 859
 860        /* CFG */
 861        region = &hdev->pci_mem_region[PCI_REGION_CFG];
 862        region->region_base = CFG_BASE;
 863        region->region_size = CFG_SIZE;
 864        region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR;
 865        region->bar_size = CFG_BAR_SIZE;
 866        region->bar_id = SRAM_CFG_BAR_ID;
 867        region->used = 1;
 868
 869        /* SRAM */
 870        region = &hdev->pci_mem_region[PCI_REGION_SRAM];
 871        region->region_base = SRAM_BASE_ADDR;
 872        region->region_size = SRAM_SIZE;
 873        region->offset_in_bar = 0;
 874        region->bar_size = CFG_BAR_SIZE;
 875        region->bar_id = SRAM_CFG_BAR_ID;
 876        region->used = 1;
 877
 878        /* DRAM */
 879        region = &hdev->pci_mem_region[PCI_REGION_DRAM];
 880        region->region_base = DRAM_PHYS_BASE;
 881        region->region_size = hdev->asic_prop.dram_size;
 882        region->offset_in_bar = 0;
 883        region->bar_size = prop->dram_pci_bar_size;
 884        region->bar_id = DDR_BAR_ID;
 885        region->used = 1;
 886}
 887
 888/*
 889 * goya_sw_init - Goya software initialization code
 890 *
 891 * @hdev: pointer to hl_device structure
 892 *
 893 */
 894static int goya_sw_init(struct hl_device *hdev)
 895{
 896        struct goya_device *goya;
 897        int rc;
 898
 899        /* Allocate device structure */
 900        goya = kzalloc(sizeof(*goya), GFP_KERNEL);
 901        if (!goya)
 902                return -ENOMEM;
 903
 904        /* according to goya_init_iatu */
 905        goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
 906
 907        goya->mme_clk = GOYA_PLL_FREQ_LOW;
 908        goya->tpc_clk = GOYA_PLL_FREQ_LOW;
 909        goya->ic_clk = GOYA_PLL_FREQ_LOW;
 910
 911        hdev->asic_specific = goya;
 912
 913        /* Create DMA pool for small allocations */
 914        hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
 915                        &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0);
 916        if (!hdev->dma_pool) {
 917                dev_err(hdev->dev, "failed to create DMA pool\n");
 918                rc = -ENOMEM;
 919                goto free_goya_device;
 920        }
 921
 922        hdev->cpu_accessible_dma_mem =
 923                        hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 924                                        HL_CPU_ACCESSIBLE_MEM_SIZE,
 925                                        &hdev->cpu_accessible_dma_address,
 926                                        GFP_KERNEL | __GFP_ZERO);
 927
 928        if (!hdev->cpu_accessible_dma_mem) {
 929                rc = -ENOMEM;
 930                goto free_dma_pool;
 931        }
 932
 933        dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n",
 934                &hdev->cpu_accessible_dma_address);
 935
 936        hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
 937        if (!hdev->cpu_accessible_dma_pool) {
 938                dev_err(hdev->dev,
 939                        "Failed to create CPU accessible DMA pool\n");
 940                rc = -ENOMEM;
 941                goto free_cpu_dma_mem;
 942        }
 943
 944        rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
 945                                (uintptr_t) hdev->cpu_accessible_dma_mem,
 946                                HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
 947        if (rc) {
 948                dev_err(hdev->dev,
 949                        "Failed to add memory to CPU accessible DMA pool\n");
 950                rc = -EFAULT;
 951                goto free_cpu_accessible_dma_pool;
 952        }
 953
 954        spin_lock_init(&goya->hw_queues_lock);
 955        hdev->supports_coresight = true;
 956        hdev->supports_soft_reset = true;
 957        hdev->allow_external_soft_reset = true;
 958
 959        goya_set_pci_memory_regions(hdev);
 960
 961        return 0;
 962
 963free_cpu_accessible_dma_pool:
 964        gen_pool_destroy(hdev->cpu_accessible_dma_pool);
 965free_cpu_dma_mem:
 966        hdev->asic_funcs->asic_dma_free_coherent(hdev,
 967                        HL_CPU_ACCESSIBLE_MEM_SIZE,
 968                        hdev->cpu_accessible_dma_mem,
 969                        hdev->cpu_accessible_dma_address);
 970free_dma_pool:
 971        dma_pool_destroy(hdev->dma_pool);
 972free_goya_device:
 973        kfree(goya);
 974
 975        return rc;
 976}
 977
 978/*
 979 * goya_sw_fini - Goya software tear-down code
 980 *
 981 * @hdev: pointer to hl_device structure
 982 *
 983 */
 984static int goya_sw_fini(struct hl_device *hdev)
 985{
 986        struct goya_device *goya = hdev->asic_specific;
 987
 988        gen_pool_destroy(hdev->cpu_accessible_dma_pool);
 989
 990        hdev->asic_funcs->asic_dma_free_coherent(hdev,
 991                        HL_CPU_ACCESSIBLE_MEM_SIZE,
 992                        hdev->cpu_accessible_dma_mem,
 993                        hdev->cpu_accessible_dma_address);
 994
 995        dma_pool_destroy(hdev->dma_pool);
 996
 997        kfree(goya);
 998
 999        return 0;
1000}
1001
1002static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
1003                dma_addr_t bus_address)
1004{
1005        struct goya_device *goya = hdev->asic_specific;
1006        u32 mtr_base_lo, mtr_base_hi;
1007        u32 so_base_lo, so_base_hi;
1008        u32 gic_base_lo, gic_base_hi;
1009        u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
1010        u32 dma_err_cfg = QMAN_DMA_ERR_MSG_EN;
1011
1012        mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1013        mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1014        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1015        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1016
1017        gic_base_lo =
1018                lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1019        gic_base_hi =
1020                upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1021
1022        WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address));
1023        WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address));
1024
1025        WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH));
1026        WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0);
1027        WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0);
1028
1029        WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1030        WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1031        WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1032        WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1033        WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1034        WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1035        WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off,
1036                        GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id);
1037
1038        /* PQ has buffer of 2 cache lines, while CQ has 8 lines */
1039        WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002);
1040        WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008);
1041
1042        if (goya->hw_cap_initialized & HW_CAP_MMU)
1043                WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_PARTLY_TRUSTED);
1044        else
1045                WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
1046
1047        if (hdev->stop_on_err)
1048                dma_err_cfg |= 1 << DMA_QM_0_GLBL_ERR_CFG_DMA_STOP_ON_ERR_SHIFT;
1049
1050        WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, dma_err_cfg);
1051        WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
1052}
1053
1054static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
1055{
1056        u32 gic_base_lo, gic_base_hi;
1057        u64 sob_addr;
1058        u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1);
1059
1060        gic_base_lo =
1061                lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1062        gic_base_hi =
1063                upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1064
1065        WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo);
1066        WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi);
1067        WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off,
1068                        GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id);
1069
1070        if (dma_id)
1071                sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
1072                                (dma_id - 1) * 4;
1073        else
1074                sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
1075
1076        WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off, upper_32_bits(sob_addr));
1077        WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001);
1078}
1079
1080/*
1081 * goya_init_dma_qmans - Initialize QMAN DMA registers
1082 *
1083 * @hdev: pointer to hl_device structure
1084 *
1085 * Initialize the H/W registers of the QMAN DMA channels
1086 *
1087 */
1088void goya_init_dma_qmans(struct hl_device *hdev)
1089{
1090        struct goya_device *goya = hdev->asic_specific;
1091        struct hl_hw_queue *q;
1092        int i;
1093
1094        if (goya->hw_cap_initialized & HW_CAP_DMA)
1095                return;
1096
1097        q = &hdev->kernel_queues[0];
1098
1099        for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
1100                q->cq_id = q->msi_vec = i;
1101                goya_init_dma_qman(hdev, i, q->bus_address);
1102                goya_init_dma_ch(hdev, i);
1103        }
1104
1105        goya->hw_cap_initialized |= HW_CAP_DMA;
1106}
1107
1108/*
1109 * goya_disable_external_queues - Disable external queues
1110 *
1111 * @hdev: pointer to hl_device structure
1112 *
1113 */
1114static void goya_disable_external_queues(struct hl_device *hdev)
1115{
1116        struct goya_device *goya = hdev->asic_specific;
1117
1118        if (!(goya->hw_cap_initialized & HW_CAP_DMA))
1119                return;
1120
1121        WREG32(mmDMA_QM_0_GLBL_CFG0, 0);
1122        WREG32(mmDMA_QM_1_GLBL_CFG0, 0);
1123        WREG32(mmDMA_QM_2_GLBL_CFG0, 0);
1124        WREG32(mmDMA_QM_3_GLBL_CFG0, 0);
1125        WREG32(mmDMA_QM_4_GLBL_CFG0, 0);
1126}
1127
1128static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
1129                                u32 cp_sts_reg, u32 glbl_sts0_reg)
1130{
1131        int rc;
1132        u32 status;
1133
1134        /* use the values of TPC0 as they are all the same*/
1135
1136        WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
1137
1138        status = RREG32(cp_sts_reg);
1139        if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) {
1140                rc = hl_poll_timeout(
1141                        hdev,
1142                        cp_sts_reg,
1143                        status,
1144                        !(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK),
1145                        1000,
1146                        QMAN_FENCE_TIMEOUT_USEC);
1147
1148                /* if QMAN is stuck in fence no need to check for stop */
1149                if (rc)
1150                        return 0;
1151        }
1152
1153        rc = hl_poll_timeout(
1154                hdev,
1155                glbl_sts0_reg,
1156                status,
1157                (status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK),
1158                1000,
1159                QMAN_STOP_TIMEOUT_USEC);
1160
1161        if (rc) {
1162                dev_err(hdev->dev,
1163                        "Timeout while waiting for QMAN to stop\n");
1164                return -EINVAL;
1165        }
1166
1167        return 0;
1168}
1169
1170/*
1171 * goya_stop_external_queues - Stop external queues
1172 *
1173 * @hdev: pointer to hl_device structure
1174 *
1175 * Returns 0 on success
1176 *
1177 */
1178static int goya_stop_external_queues(struct hl_device *hdev)
1179{
1180        int rc, retval = 0;
1181
1182        struct goya_device *goya = hdev->asic_specific;
1183
1184        if (!(goya->hw_cap_initialized & HW_CAP_DMA))
1185                return retval;
1186
1187        rc = goya_stop_queue(hdev,
1188                        mmDMA_QM_0_GLBL_CFG1,
1189                        mmDMA_QM_0_CP_STS,
1190                        mmDMA_QM_0_GLBL_STS0);
1191
1192        if (rc) {
1193                dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
1194                retval = -EIO;
1195        }
1196
1197        rc = goya_stop_queue(hdev,
1198                        mmDMA_QM_1_GLBL_CFG1,
1199                        mmDMA_QM_1_CP_STS,
1200                        mmDMA_QM_1_GLBL_STS0);
1201
1202        if (rc) {
1203                dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
1204                retval = -EIO;
1205        }
1206
1207        rc = goya_stop_queue(hdev,
1208                        mmDMA_QM_2_GLBL_CFG1,
1209                        mmDMA_QM_2_CP_STS,
1210                        mmDMA_QM_2_GLBL_STS0);
1211
1212        if (rc) {
1213                dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
1214                retval = -EIO;
1215        }
1216
1217        rc = goya_stop_queue(hdev,
1218                        mmDMA_QM_3_GLBL_CFG1,
1219                        mmDMA_QM_3_CP_STS,
1220                        mmDMA_QM_3_GLBL_STS0);
1221
1222        if (rc) {
1223                dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
1224                retval = -EIO;
1225        }
1226
1227        rc = goya_stop_queue(hdev,
1228                        mmDMA_QM_4_GLBL_CFG1,
1229                        mmDMA_QM_4_CP_STS,
1230                        mmDMA_QM_4_GLBL_STS0);
1231
1232        if (rc) {
1233                dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
1234                retval = -EIO;
1235        }
1236
1237        return retval;
1238}
1239
1240/*
1241 * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
1242 *
1243 * @hdev: pointer to hl_device structure
1244 *
1245 * Returns 0 on success
1246 *
1247 */
1248int goya_init_cpu_queues(struct hl_device *hdev)
1249{
1250        struct goya_device *goya = hdev->asic_specific;
1251        struct asic_fixed_properties *prop = &hdev->asic_prop;
1252        struct hl_eq *eq;
1253        u32 status;
1254        struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
1255        int err;
1256
1257        if (!hdev->cpu_queues_enable)
1258                return 0;
1259
1260        if (goya->hw_cap_initialized & HW_CAP_CPU_Q)
1261                return 0;
1262
1263        eq = &hdev->event_queue;
1264
1265        WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
1266        WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
1267
1268        WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
1269        WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
1270
1271        WREG32(mmCPU_CQ_BASE_ADDR_LOW,
1272                        lower_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
1273        WREG32(mmCPU_CQ_BASE_ADDR_HIGH,
1274                        upper_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
1275
1276        WREG32(mmCPU_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
1277        WREG32(mmCPU_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
1278        WREG32(mmCPU_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
1279
1280        /* Used for EQ CI */
1281        WREG32(mmCPU_EQ_CI, 0);
1282
1283        WREG32(mmCPU_IF_PF_PQ_PI, 0);
1284
1285        WREG32(mmCPU_PQ_INIT_STATUS, PQ_INIT_STATUS_READY_FOR_CP);
1286
1287        WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
1288                        GOYA_ASYNC_EVENT_ID_PI_UPDATE);
1289
1290        err = hl_poll_timeout(
1291                hdev,
1292                mmCPU_PQ_INIT_STATUS,
1293                status,
1294                (status == PQ_INIT_STATUS_READY_FOR_HOST),
1295                1000,
1296                GOYA_CPU_TIMEOUT_USEC);
1297
1298        if (err) {
1299                dev_err(hdev->dev,
1300                        "Failed to setup communication with device CPU\n");
1301                return -EIO;
1302        }
1303
1304        /* update FW application security bits */
1305        if (prop->fw_cpu_boot_dev_sts0_valid)
1306                prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);
1307
1308        if (prop->fw_cpu_boot_dev_sts1_valid)
1309                prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);
1310
1311        goya->hw_cap_initialized |= HW_CAP_CPU_Q;
1312        return 0;
1313}
1314
1315static void goya_set_pll_refclk(struct hl_device *hdev)
1316{
1317        WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);
1318        WREG32(mmCPU_PLL_DIV_SEL_1, 0x0);
1319        WREG32(mmCPU_PLL_DIV_SEL_2, 0x0);
1320        WREG32(mmCPU_PLL_DIV_SEL_3, 0x0);
1321
1322        WREG32(mmIC_PLL_DIV_SEL_0, 0x0);
1323        WREG32(mmIC_PLL_DIV_SEL_1, 0x0);
1324        WREG32(mmIC_PLL_DIV_SEL_2, 0x0);
1325        WREG32(mmIC_PLL_DIV_SEL_3, 0x0);
1326
1327        WREG32(mmMC_PLL_DIV_SEL_0, 0x0);
1328        WREG32(mmMC_PLL_DIV_SEL_1, 0x0);
1329        WREG32(mmMC_PLL_DIV_SEL_2, 0x0);
1330        WREG32(mmMC_PLL_DIV_SEL_3, 0x0);
1331
1332        WREG32(mmPSOC_MME_PLL_DIV_SEL_0, 0x0);
1333        WREG32(mmPSOC_MME_PLL_DIV_SEL_1, 0x0);
1334        WREG32(mmPSOC_MME_PLL_DIV_SEL_2, 0x0);
1335        WREG32(mmPSOC_MME_PLL_DIV_SEL_3, 0x0);
1336
1337        WREG32(mmPSOC_PCI_PLL_DIV_SEL_0, 0x0);
1338        WREG32(mmPSOC_PCI_PLL_DIV_SEL_1, 0x0);
1339        WREG32(mmPSOC_PCI_PLL_DIV_SEL_2, 0x0);
1340        WREG32(mmPSOC_PCI_PLL_DIV_SEL_3, 0x0);
1341
1342        WREG32(mmPSOC_EMMC_PLL_DIV_SEL_0, 0x0);
1343        WREG32(mmPSOC_EMMC_PLL_DIV_SEL_1, 0x0);
1344        WREG32(mmPSOC_EMMC_PLL_DIV_SEL_2, 0x0);
1345        WREG32(mmPSOC_EMMC_PLL_DIV_SEL_3, 0x0);
1346
1347        WREG32(mmTPC_PLL_DIV_SEL_0, 0x0);
1348        WREG32(mmTPC_PLL_DIV_SEL_1, 0x0);
1349        WREG32(mmTPC_PLL_DIV_SEL_2, 0x0);
1350        WREG32(mmTPC_PLL_DIV_SEL_3, 0x0);
1351}
1352
1353static void goya_disable_clk_rlx(struct hl_device *hdev)
1354{
1355        WREG32(mmPSOC_MME_PLL_CLK_RLX_0, 0x100010);
1356        WREG32(mmIC_PLL_CLK_RLX_0, 0x100010);
1357}
1358
1359static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id)
1360{
1361        u64 tpc_eml_address;
1362        u32 val, tpc_offset, tpc_eml_offset, tpc_slm_offset;
1363        int err, slm_index;
1364
1365        tpc_offset = tpc_id * 0x40000;
1366        tpc_eml_offset = tpc_id * 0x200000;
1367        tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE);
1368        tpc_slm_offset = tpc_eml_address + 0x100000;
1369
1370        /*
1371         * Workaround for Bug H2 #2443 :
1372         * "TPC SB is not initialized on chip reset"
1373         */
1374
1375        val = RREG32(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset);
1376        if (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_MASK)
1377                dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n",
1378                        tpc_id);
1379
1380        WREG32(mmTPC0_CFG_FUNC_MBIST_PAT + tpc_offset, val & 0xFFFFF000);
1381
1382        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_0 + tpc_offset, 0x37FF);
1383        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_1 + tpc_offset, 0x303F);
1384        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_2 + tpc_offset, 0x71FF);
1385        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_3 + tpc_offset, 0x71FF);
1386        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_4 + tpc_offset, 0x70FF);
1387        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_5 + tpc_offset, 0x70FF);
1388        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_6 + tpc_offset, 0x70FF);
1389        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_7 + tpc_offset, 0x70FF);
1390        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_8 + tpc_offset, 0x70FF);
1391        WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_9 + tpc_offset, 0x70FF);
1392
1393        WREG32_OR(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
1394                1 << TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_SHIFT);
1395
1396        err = hl_poll_timeout(
1397                hdev,
1398                mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
1399                val,
1400                (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_MASK),
1401                1000,
1402                HL_DEVICE_TIMEOUT_USEC);
1403
1404        if (err)
1405                dev_err(hdev->dev,
1406                        "Timeout while waiting for TPC%d MBIST DONE\n", tpc_id);
1407
1408        WREG32_OR(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
1409                1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT);
1410
1411        msleep(GOYA_RESET_WAIT_MSEC);
1412
1413        WREG32_AND(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
1414                ~(1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT));
1415
1416        msleep(GOYA_RESET_WAIT_MSEC);
1417
1418        for (slm_index = 0 ; slm_index < 256 ; slm_index++)
1419                WREG32(tpc_slm_offset + (slm_index << 2), 0);
1420
1421        val = RREG32(tpc_slm_offset);
1422}
1423
1424static void goya_tpc_mbist_workaround(struct hl_device *hdev)
1425{
1426        struct goya_device *goya = hdev->asic_specific;
1427        int i;
1428
1429        if (hdev->pldm)
1430                return;
1431
1432        if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST)
1433                return;
1434
1435        /* Workaround for H2 #2443 */
1436
1437        for (i = 0 ; i < TPC_MAX_NUM ; i++)
1438                _goya_tpc_mbist_workaround(hdev, i);
1439
1440        goya->hw_cap_initialized |= HW_CAP_TPC_MBIST;
1441}
1442
1443/*
1444 * goya_init_golden_registers - Initialize golden registers
1445 *
1446 * @hdev: pointer to hl_device structure
1447 *
1448 * Initialize the H/W registers of the device
1449 *
1450 */
1451static void goya_init_golden_registers(struct hl_device *hdev)
1452{
1453        struct goya_device *goya = hdev->asic_specific;
1454        u32 polynom[10], tpc_intr_mask, offset;
1455        int i;
1456
1457        if (goya->hw_cap_initialized & HW_CAP_GOLDEN)
1458                return;
1459
1460        polynom[0] = 0x00020080;
1461        polynom[1] = 0x00401000;
1462        polynom[2] = 0x00200800;
1463        polynom[3] = 0x00002000;
1464        polynom[4] = 0x00080200;
1465        polynom[5] = 0x00040100;
1466        polynom[6] = 0x00100400;
1467        polynom[7] = 0x00004000;
1468        polynom[8] = 0x00010000;
1469        polynom[9] = 0x00008000;
1470
1471        /* Mask all arithmetic interrupts from TPC */
1472        tpc_intr_mask = 0x7FFF;
1473
1474        for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x20000) {
1475                WREG32(mmSRAM_Y0_X0_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1476                WREG32(mmSRAM_Y0_X1_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1477                WREG32(mmSRAM_Y0_X2_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1478                WREG32(mmSRAM_Y0_X3_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1479                WREG32(mmSRAM_Y0_X4_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1480
1481                WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_L_ARB + offset, 0x204);
1482                WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_L_ARB + offset, 0x204);
1483                WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_L_ARB + offset, 0x204);
1484                WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_L_ARB + offset, 0x204);
1485                WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_L_ARB + offset, 0x204);
1486
1487
1488                WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_E_ARB + offset, 0x206);
1489                WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_E_ARB + offset, 0x206);
1490                WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_E_ARB + offset, 0x206);
1491                WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_E_ARB + offset, 0x207);
1492                WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_E_ARB + offset, 0x207);
1493
1494                WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_W_ARB + offset, 0x207);
1495                WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_W_ARB + offset, 0x207);
1496                WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_W_ARB + offset, 0x206);
1497                WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_W_ARB + offset, 0x206);
1498                WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_W_ARB + offset, 0x206);
1499
1500                WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_E_ARB + offset, 0x101);
1501                WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_E_ARB + offset, 0x102);
1502                WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_E_ARB + offset, 0x103);
1503                WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_E_ARB + offset, 0x104);
1504                WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_E_ARB + offset, 0x105);
1505
1506                WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_W_ARB + offset, 0x105);
1507                WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_W_ARB + offset, 0x104);
1508                WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_W_ARB + offset, 0x103);
1509                WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_W_ARB + offset, 0x102);
1510                WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_W_ARB + offset, 0x101);
1511        }
1512
1513        WREG32(mmMME_STORE_MAX_CREDIT, 0x21);
1514        WREG32(mmMME_AGU, 0x0f0f0f10);
1515        WREG32(mmMME_SEI_MASK, ~0x0);
1516
1517        WREG32(mmMME6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1518        WREG32(mmMME5_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
1519        WREG32(mmMME4_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
1520        WREG32(mmMME3_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
1521        WREG32(mmMME2_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1522        WREG32(mmMME1_RTR_HBW_RD_RQ_N_ARB, 0x07010701);
1523        WREG32(mmMME6_RTR_HBW_RD_RQ_S_ARB, 0x04010401);
1524        WREG32(mmMME5_RTR_HBW_RD_RQ_S_ARB, 0x04050401);
1525        WREG32(mmMME4_RTR_HBW_RD_RQ_S_ARB, 0x03070301);
1526        WREG32(mmMME3_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
1527        WREG32(mmMME2_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
1528        WREG32(mmMME1_RTR_HBW_RD_RQ_S_ARB, 0x01050105);
1529        WREG32(mmMME6_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
1530        WREG32(mmMME5_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
1531        WREG32(mmMME4_RTR_HBW_RD_RQ_W_ARB, 0x01040301);
1532        WREG32(mmMME3_RTR_HBW_RD_RQ_W_ARB, 0x01030401);
1533        WREG32(mmMME2_RTR_HBW_RD_RQ_W_ARB, 0x01040101);
1534        WREG32(mmMME1_RTR_HBW_RD_RQ_W_ARB, 0x01050101);
1535        WREG32(mmMME6_RTR_HBW_WR_RQ_N_ARB, 0x02020202);
1536        WREG32(mmMME5_RTR_HBW_WR_RQ_N_ARB, 0x01070101);
1537        WREG32(mmMME4_RTR_HBW_WR_RQ_N_ARB, 0x02020201);
1538        WREG32(mmMME3_RTR_HBW_WR_RQ_N_ARB, 0x07020701);
1539        WREG32(mmMME2_RTR_HBW_WR_RQ_N_ARB, 0x01020101);
1540        WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1541        WREG32(mmMME6_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1542        WREG32(mmMME5_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1543        WREG32(mmMME4_RTR_HBW_WR_RQ_S_ARB, 0x07020701);
1544        WREG32(mmMME3_RTR_HBW_WR_RQ_S_ARB, 0x02020201);
1545        WREG32(mmMME2_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1546        WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01020102);
1547        WREG32(mmMME6_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
1548        WREG32(mmMME5_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
1549        WREG32(mmMME4_RTR_HBW_WR_RQ_W_ARB, 0x07020707);
1550        WREG32(mmMME3_RTR_HBW_WR_RQ_W_ARB, 0x01020201);
1551        WREG32(mmMME2_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
1552        WREG32(mmMME1_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
1553        WREG32(mmMME6_RTR_HBW_RD_RS_N_ARB, 0x01070102);
1554        WREG32(mmMME5_RTR_HBW_RD_RS_N_ARB, 0x01070102);
1555        WREG32(mmMME4_RTR_HBW_RD_RS_N_ARB, 0x01060102);
1556        WREG32(mmMME3_RTR_HBW_RD_RS_N_ARB, 0x01040102);
1557        WREG32(mmMME2_RTR_HBW_RD_RS_N_ARB, 0x01020102);
1558        WREG32(mmMME1_RTR_HBW_RD_RS_N_ARB, 0x01020107);
1559        WREG32(mmMME6_RTR_HBW_RD_RS_S_ARB, 0x01020106);
1560        WREG32(mmMME5_RTR_HBW_RD_RS_S_ARB, 0x01020102);
1561        WREG32(mmMME4_RTR_HBW_RD_RS_S_ARB, 0x01040102);
1562        WREG32(mmMME3_RTR_HBW_RD_RS_S_ARB, 0x01060102);
1563        WREG32(mmMME2_RTR_HBW_RD_RS_S_ARB, 0x01070102);
1564        WREG32(mmMME1_RTR_HBW_RD_RS_S_ARB, 0x01070102);
1565        WREG32(mmMME6_RTR_HBW_RD_RS_E_ARB, 0x01020702);
1566        WREG32(mmMME5_RTR_HBW_RD_RS_E_ARB, 0x01020702);
1567        WREG32(mmMME4_RTR_HBW_RD_RS_E_ARB, 0x01040602);
1568        WREG32(mmMME3_RTR_HBW_RD_RS_E_ARB, 0x01060402);
1569        WREG32(mmMME2_RTR_HBW_RD_RS_E_ARB, 0x01070202);
1570        WREG32(mmMME1_RTR_HBW_RD_RS_E_ARB, 0x01070102);
1571        WREG32(mmMME6_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1572        WREG32(mmMME5_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1573        WREG32(mmMME4_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1574        WREG32(mmMME3_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1575        WREG32(mmMME2_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1576        WREG32(mmMME1_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1577        WREG32(mmMME6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
1578        WREG32(mmMME5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
1579        WREG32(mmMME4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
1580        WREG32(mmMME3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
1581        WREG32(mmMME2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1582        WREG32(mmMME1_RTR_HBW_WR_RS_N_ARB, 0x01010107);
1583        WREG32(mmMME6_RTR_HBW_WR_RS_S_ARB, 0x01010107);
1584        WREG32(mmMME5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1585        WREG32(mmMME4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
1586        WREG32(mmMME3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
1587        WREG32(mmMME2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
1588        WREG32(mmMME1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
1589        WREG32(mmMME6_RTR_HBW_WR_RS_E_ARB, 0x01010501);
1590        WREG32(mmMME5_RTR_HBW_WR_RS_E_ARB, 0x01010501);
1591        WREG32(mmMME4_RTR_HBW_WR_RS_E_ARB, 0x01040301);
1592        WREG32(mmMME3_RTR_HBW_WR_RS_E_ARB, 0x01030401);
1593        WREG32(mmMME2_RTR_HBW_WR_RS_E_ARB, 0x01040101);
1594        WREG32(mmMME1_RTR_HBW_WR_RS_E_ARB, 0x01050101);
1595        WREG32(mmMME6_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1596        WREG32(mmMME5_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1597        WREG32(mmMME4_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1598        WREG32(mmMME3_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1599        WREG32(mmMME2_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1600        WREG32(mmMME1_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1601
1602        WREG32(mmTPC1_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1603        WREG32(mmTPC1_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
1604        WREG32(mmTPC1_RTR_HBW_RD_RQ_E_ARB, 0x01060101);
1605        WREG32(mmTPC1_RTR_HBW_WR_RQ_N_ARB, 0x02020102);
1606        WREG32(mmTPC1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1607        WREG32(mmTPC1_RTR_HBW_WR_RQ_E_ARB, 0x02070202);
1608        WREG32(mmTPC1_RTR_HBW_RD_RS_N_ARB, 0x01020201);
1609        WREG32(mmTPC1_RTR_HBW_RD_RS_S_ARB, 0x01070201);
1610        WREG32(mmTPC1_RTR_HBW_RD_RS_W_ARB, 0x01070202);
1611        WREG32(mmTPC1_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1612        WREG32(mmTPC1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
1613        WREG32(mmTPC1_RTR_HBW_WR_RS_W_ARB, 0x01050101);
1614
1615        WREG32(mmTPC2_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
1616        WREG32(mmTPC2_RTR_HBW_RD_RQ_S_ARB, 0x01050101);
1617        WREG32(mmTPC2_RTR_HBW_RD_RQ_E_ARB, 0x01010201);
1618        WREG32(mmTPC2_RTR_HBW_WR_RQ_N_ARB, 0x02040102);
1619        WREG32(mmTPC2_RTR_HBW_WR_RQ_S_ARB, 0x01050101);
1620        WREG32(mmTPC2_RTR_HBW_WR_RQ_E_ARB, 0x02060202);
1621        WREG32(mmTPC2_RTR_HBW_RD_RS_N_ARB, 0x01020201);
1622        WREG32(mmTPC2_RTR_HBW_RD_RS_S_ARB, 0x01070201);
1623        WREG32(mmTPC2_RTR_HBW_RD_RS_W_ARB, 0x01070202);
1624        WREG32(mmTPC2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1625        WREG32(mmTPC2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
1626        WREG32(mmTPC2_RTR_HBW_WR_RS_W_ARB, 0x01040101);
1627
1628        WREG32(mmTPC3_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
1629        WREG32(mmTPC3_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
1630        WREG32(mmTPC3_RTR_HBW_RD_RQ_E_ARB, 0x01040301);
1631        WREG32(mmTPC3_RTR_HBW_WR_RQ_N_ARB, 0x02060102);
1632        WREG32(mmTPC3_RTR_HBW_WR_RQ_S_ARB, 0x01040101);
1633        WREG32(mmTPC3_RTR_HBW_WR_RQ_E_ARB, 0x01040301);
1634        WREG32(mmTPC3_RTR_HBW_RD_RS_N_ARB, 0x01040201);
1635        WREG32(mmTPC3_RTR_HBW_RD_RS_S_ARB, 0x01060201);
1636        WREG32(mmTPC3_RTR_HBW_RD_RS_W_ARB, 0x01060402);
1637        WREG32(mmTPC3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
1638        WREG32(mmTPC3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
1639        WREG32(mmTPC3_RTR_HBW_WR_RS_W_ARB, 0x01030401);
1640
1641        WREG32(mmTPC4_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
1642        WREG32(mmTPC4_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
1643        WREG32(mmTPC4_RTR_HBW_RD_RQ_E_ARB, 0x01030401);
1644        WREG32(mmTPC4_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
1645        WREG32(mmTPC4_RTR_HBW_WR_RQ_S_ARB, 0x01030101);
1646        WREG32(mmTPC4_RTR_HBW_WR_RQ_E_ARB, 0x02060702);
1647        WREG32(mmTPC4_RTR_HBW_RD_RS_N_ARB, 0x01060201);
1648        WREG32(mmTPC4_RTR_HBW_RD_RS_S_ARB, 0x01040201);
1649        WREG32(mmTPC4_RTR_HBW_RD_RS_W_ARB, 0x01040602);
1650        WREG32(mmTPC4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
1651        WREG32(mmTPC4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
1652        WREG32(mmTPC4_RTR_HBW_WR_RS_W_ARB, 0x01040301);
1653
1654        WREG32(mmTPC5_RTR_HBW_RD_RQ_N_ARB, 0x01050101);
1655        WREG32(mmTPC5_RTR_HBW_RD_RQ_S_ARB, 0x01020101);
1656        WREG32(mmTPC5_RTR_HBW_RD_RQ_E_ARB, 0x01200501);
1657        WREG32(mmTPC5_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
1658        WREG32(mmTPC5_RTR_HBW_WR_RQ_S_ARB, 0x01020101);
1659        WREG32(mmTPC5_RTR_HBW_WR_RQ_E_ARB, 0x02020602);
1660        WREG32(mmTPC5_RTR_HBW_RD_RS_N_ARB, 0x01070201);
1661        WREG32(mmTPC5_RTR_HBW_RD_RS_S_ARB, 0x01020201);
1662        WREG32(mmTPC5_RTR_HBW_RD_RS_W_ARB, 0x01020702);
1663        WREG32(mmTPC5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
1664        WREG32(mmTPC5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1665        WREG32(mmTPC5_RTR_HBW_WR_RS_W_ARB, 0x01010501);
1666
1667        WREG32(mmTPC6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1668        WREG32(mmTPC6_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
1669        WREG32(mmTPC6_RTR_HBW_RD_RQ_E_ARB, 0x01010601);
1670        WREG32(mmTPC6_RTR_HBW_WR_RQ_N_ARB, 0x01010101);
1671        WREG32(mmTPC6_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1672        WREG32(mmTPC6_RTR_HBW_WR_RQ_E_ARB, 0x02020702);
1673        WREG32(mmTPC6_RTR_HBW_RD_RS_N_ARB, 0x01010101);
1674        WREG32(mmTPC6_RTR_HBW_RD_RS_S_ARB, 0x01010101);
1675        WREG32(mmTPC6_RTR_HBW_RD_RS_W_ARB, 0x01020702);
1676        WREG32(mmTPC6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
1677        WREG32(mmTPC6_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1678        WREG32(mmTPC6_RTR_HBW_WR_RS_W_ARB, 0x01010501);
1679
1680        for (i = 0, offset = 0 ; i < 10 ; i++, offset += 4) {
1681                WREG32(mmMME1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1682                WREG32(mmMME2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1683                WREG32(mmMME3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1684                WREG32(mmMME4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1685                WREG32(mmMME5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1686                WREG32(mmMME6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1687
1688                WREG32(mmTPC0_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1689                WREG32(mmTPC1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1690                WREG32(mmTPC2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1691                WREG32(mmTPC3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1692                WREG32(mmTPC4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1693                WREG32(mmTPC5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1694                WREG32(mmTPC6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1695                WREG32(mmTPC7_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1696
1697                WREG32(mmPCI_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1698                WREG32(mmDMA_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1699        }
1700
1701        for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x40000) {
1702                WREG32(mmMME1_RTR_SCRAMB_EN + offset,
1703                                1 << MME1_RTR_SCRAMB_EN_VAL_SHIFT);
1704                WREG32(mmMME1_RTR_NON_LIN_SCRAMB + offset,
1705                                1 << MME1_RTR_NON_LIN_SCRAMB_EN_SHIFT);
1706        }
1707
1708        for (i = 0, offset = 0 ; i < 8 ; i++, offset += 0x40000) {
1709                /*
1710                 * Workaround for Bug H2 #2441 :
1711                 * "ST.NOP set trace event illegal opcode"
1712                 */
1713                WREG32(mmTPC0_CFG_TPC_INTR_MASK + offset, tpc_intr_mask);
1714
1715                WREG32(mmTPC0_NRTR_SCRAMB_EN + offset,
1716                                1 << TPC0_NRTR_SCRAMB_EN_VAL_SHIFT);
1717                WREG32(mmTPC0_NRTR_NON_LIN_SCRAMB + offset,
1718                                1 << TPC0_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1719
1720                WREG32_FIELD(TPC0_CFG_MSS_CONFIG, offset,
1721                                ICACHE_FETCH_LINE_NUM, 2);
1722        }
1723
1724        WREG32(mmDMA_NRTR_SCRAMB_EN, 1 << DMA_NRTR_SCRAMB_EN_VAL_SHIFT);
1725        WREG32(mmDMA_NRTR_NON_LIN_SCRAMB,
1726                        1 << DMA_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1727
1728        WREG32(mmPCI_NRTR_SCRAMB_EN, 1 << PCI_NRTR_SCRAMB_EN_VAL_SHIFT);
1729        WREG32(mmPCI_NRTR_NON_LIN_SCRAMB,
1730                        1 << PCI_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1731
1732        /*
1733         * Workaround for H2 #HW-23 bug
1734         * Set DMA max outstanding read requests to 240 on DMA CH 1.
1735         * This limitation is still large enough to not affect Gen4 bandwidth.
1736         * We need to only limit that DMA channel because the user can only read
1737         * from Host using DMA CH 1
1738         */
1739        WREG32(mmDMA_CH_1_CFG0, 0x0fff00F0);
1740
1741        WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
1742
1743        goya->hw_cap_initialized |= HW_CAP_GOLDEN;
1744}
1745
1746static void goya_init_mme_qman(struct hl_device *hdev)
1747{
1748        u32 mtr_base_lo, mtr_base_hi;
1749        u32 so_base_lo, so_base_hi;
1750        u32 gic_base_lo, gic_base_hi;
1751        u64 qman_base_addr;
1752
1753        mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1754        mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1755        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1756        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1757
1758        gic_base_lo =
1759                lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1760        gic_base_hi =
1761                upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1762
1763        qman_base_addr = hdev->asic_prop.sram_base_address +
1764                                MME_QMAN_BASE_OFFSET;
1765
1766        WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr));
1767        WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr));
1768        WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH));
1769        WREG32(mmMME_QM_PQ_PI, 0);
1770        WREG32(mmMME_QM_PQ_CI, 0);
1771        WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0);
1772        WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4);
1773        WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8);
1774        WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC);
1775
1776        WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1777        WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1778        WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1779        WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1780
1781        /* QMAN CQ has 8 cache lines */
1782        WREG32(mmMME_QM_CQ_CFG1, 0x00080008);
1783
1784        WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo);
1785        WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi);
1786
1787        WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM);
1788
1789        WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN);
1790
1791        WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT);
1792
1793        WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE);
1794}
1795
1796static void goya_init_mme_cmdq(struct hl_device *hdev)
1797{
1798        u32 mtr_base_lo, mtr_base_hi;
1799        u32 so_base_lo, so_base_hi;
1800        u32 gic_base_lo, gic_base_hi;
1801
1802        mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1803        mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1804        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1805        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1806
1807        gic_base_lo =
1808                lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1809        gic_base_hi =
1810                upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1811
1812        WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1813        WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1814        WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1815        WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1816
1817        /* CMDQ CQ has 20 cache lines */
1818        WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014);
1819
1820        WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo);
1821        WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi);
1822
1823        WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ);
1824
1825        WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN);
1826
1827        WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT);
1828
1829        WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
1830}
1831
1832void goya_init_mme_qmans(struct hl_device *hdev)
1833{
1834        struct goya_device *goya = hdev->asic_specific;
1835        u32 so_base_lo, so_base_hi;
1836
1837        if (goya->hw_cap_initialized & HW_CAP_MME)
1838                return;
1839
1840        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1841        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1842
1843        WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo);
1844        WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi);
1845
1846        goya_init_mme_qman(hdev);
1847        goya_init_mme_cmdq(hdev);
1848
1849        goya->hw_cap_initialized |= HW_CAP_MME;
1850}
1851
1852static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
1853{
1854        u32 mtr_base_lo, mtr_base_hi;
1855        u32 so_base_lo, so_base_hi;
1856        u32 gic_base_lo, gic_base_hi;
1857        u64 qman_base_addr;
1858        u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI);
1859
1860        mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1861        mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1862        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1863        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1864
1865        gic_base_lo =
1866                lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1867        gic_base_hi =
1868                upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1869
1870        qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
1871
1872        WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr));
1873        WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr));
1874        WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH));
1875        WREG32(mmTPC0_QM_PQ_PI + reg_off, 0);
1876        WREG32(mmTPC0_QM_PQ_CI + reg_off, 0);
1877        WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0);
1878        WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4);
1879        WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8);
1880        WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC);
1881
1882        WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1883        WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1884        WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1885        WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1886
1887        WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008);
1888
1889        WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1890        WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1891
1892        WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off,
1893                        GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id);
1894
1895        WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN);
1896
1897        WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT);
1898
1899        WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE);
1900}
1901
1902static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
1903{
1904        u32 mtr_base_lo, mtr_base_hi;
1905        u32 so_base_lo, so_base_hi;
1906        u32 gic_base_lo, gic_base_hi;
1907        u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1);
1908
1909        mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1910        mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1911        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1912        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1913
1914        gic_base_lo =
1915                lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1916        gic_base_hi =
1917                upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1918
1919        WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1920        WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1921        WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1922        WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1923
1924        WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014);
1925
1926        WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1927        WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1928
1929        WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off,
1930                        GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id);
1931
1932        WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN);
1933
1934        WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT);
1935
1936        WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
1937}
1938
1939void goya_init_tpc_qmans(struct hl_device *hdev)
1940{
1941        struct goya_device *goya = hdev->asic_specific;
1942        u32 so_base_lo, so_base_hi;
1943        u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW -
1944                        mmTPC0_CFG_SM_BASE_ADDRESS_LOW;
1945        int i;
1946
1947        if (goya->hw_cap_initialized & HW_CAP_TPC)
1948                return;
1949
1950        so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1951        so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1952
1953        for (i = 0 ; i < TPC_MAX_NUM ; i++) {
1954                WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off,
1955                                so_base_lo);
1956                WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off,
1957                                so_base_hi);
1958        }
1959
1960        goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
1961        goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
1962        goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
1963        goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
1964        goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
1965        goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
1966        goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
1967        goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
1968
1969        for (i = 0 ; i < TPC_MAX_NUM ; i++)
1970                goya_init_tpc_cmdq(hdev, i);
1971
1972        goya->hw_cap_initialized |= HW_CAP_TPC;
1973}
1974
1975/*
1976 * goya_disable_internal_queues - Disable internal queues
1977 *
1978 * @hdev: pointer to hl_device structure
1979 *
1980 */
1981static void goya_disable_internal_queues(struct hl_device *hdev)
1982{
1983        struct goya_device *goya = hdev->asic_specific;
1984
1985        if (!(goya->hw_cap_initialized & HW_CAP_MME))
1986                goto disable_tpc;
1987
1988        WREG32(mmMME_QM_GLBL_CFG0, 0);
1989        WREG32(mmMME_CMDQ_GLBL_CFG0, 0);
1990
1991disable_tpc:
1992        if (!(goya->hw_cap_initialized & HW_CAP_TPC))
1993                return;
1994
1995        WREG32(mmTPC0_QM_GLBL_CFG0, 0);
1996        WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0);
1997
1998        WREG32(mmTPC1_QM_GLBL_CFG0, 0);
1999        WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0);
2000
2001        WREG32(mmTPC2_QM_GLBL_CFG0, 0);
2002        WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0);
2003
2004        WREG32(mmTPC3_QM_GLBL_CFG0, 0);
2005        WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0);
2006
2007        WREG32(mmTPC4_QM_GLBL_CFG0, 0);
2008        WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0);
2009
2010        WREG32(mmTPC5_QM_GLBL_CFG0, 0);
2011        WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0);
2012
2013        WREG32(mmTPC6_QM_GLBL_CFG0, 0);
2014        WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0);
2015
2016        WREG32(mmTPC7_QM_GLBL_CFG0, 0);
2017        WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0);
2018}
2019
2020/*
2021 * goya_stop_internal_queues - Stop internal queues
2022 *
2023 * @hdev: pointer to hl_device structure
2024 *
2025 * Returns 0 on success
2026 *
2027 */
2028static int goya_stop_internal_queues(struct hl_device *hdev)
2029{
2030        struct goya_device *goya = hdev->asic_specific;
2031        int rc, retval = 0;
2032
2033        if (!(goya->hw_cap_initialized & HW_CAP_MME))
2034                goto stop_tpc;
2035
2036        /*
2037         * Each queue (QMAN) is a separate H/W logic. That means that each
2038         * QMAN can be stopped independently and failure to stop one does NOT
2039         * mandate we should not try to stop other QMANs
2040         */
2041
2042        rc = goya_stop_queue(hdev,
2043                        mmMME_QM_GLBL_CFG1,
2044                        mmMME_QM_CP_STS,
2045                        mmMME_QM_GLBL_STS0);
2046
2047        if (rc) {
2048                dev_err(hdev->dev, "failed to stop MME QMAN\n");
2049                retval = -EIO;
2050        }
2051
2052        rc = goya_stop_queue(hdev,
2053                        mmMME_CMDQ_GLBL_CFG1,
2054                        mmMME_CMDQ_CP_STS,
2055                        mmMME_CMDQ_GLBL_STS0);
2056
2057        if (rc) {
2058                dev_err(hdev->dev, "failed to stop MME CMDQ\n");
2059                retval = -EIO;
2060        }
2061
2062stop_tpc:
2063        if (!(goya->hw_cap_initialized & HW_CAP_TPC))
2064                return retval;
2065
2066        rc = goya_stop_queue(hdev,
2067                        mmTPC0_QM_GLBL_CFG1,
2068                        mmTPC0_QM_CP_STS,
2069                        mmTPC0_QM_GLBL_STS0);
2070
2071        if (rc) {
2072                dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
2073                retval = -EIO;
2074        }
2075
2076        rc = goya_stop_queue(hdev,
2077                        mmTPC0_CMDQ_GLBL_CFG1,
2078                        mmTPC0_CMDQ_CP_STS,
2079                        mmTPC0_CMDQ_GLBL_STS0);
2080
2081        if (rc) {
2082                dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
2083                retval = -EIO;
2084        }
2085
2086        rc = goya_stop_queue(hdev,
2087                        mmTPC1_QM_GLBL_CFG1,
2088                        mmTPC1_QM_CP_STS,
2089                        mmTPC1_QM_GLBL_STS0);
2090
2091        if (rc) {
2092                dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
2093                retval = -EIO;
2094        }
2095
2096        rc = goya_stop_queue(hdev,
2097                        mmTPC1_CMDQ_GLBL_CFG1,
2098                        mmTPC1_CMDQ_CP_STS,
2099                        mmTPC1_CMDQ_GLBL_STS0);
2100
2101        if (rc) {
2102                dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
2103                retval = -EIO;
2104        }
2105
2106        rc = goya_stop_queue(hdev,
2107                        mmTPC2_QM_GLBL_CFG1,
2108                        mmTPC2_QM_CP_STS,
2109                        mmTPC2_QM_GLBL_STS0);
2110
2111        if (rc) {
2112                dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
2113                retval = -EIO;
2114        }
2115
2116        rc = goya_stop_queue(hdev,
2117                        mmTPC2_CMDQ_GLBL_CFG1,
2118                        mmTPC2_CMDQ_CP_STS,
2119                        mmTPC2_CMDQ_GLBL_STS0);
2120
2121        if (rc) {
2122                dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
2123                retval = -EIO;
2124        }
2125
2126        rc = goya_stop_queue(hdev,
2127                        mmTPC3_QM_GLBL_CFG1,
2128                        mmTPC3_QM_CP_STS,
2129                        mmTPC3_QM_GLBL_STS0);
2130
2131        if (rc) {
2132                dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
2133                retval = -EIO;
2134        }
2135
2136        rc = goya_stop_queue(hdev,
2137                        mmTPC3_CMDQ_GLBL_CFG1,
2138                        mmTPC3_CMDQ_CP_STS,
2139                        mmTPC3_CMDQ_GLBL_STS0);
2140
2141        if (rc) {
2142                dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
2143                retval = -EIO;
2144        }
2145
2146        rc = goya_stop_queue(hdev,
2147                        mmTPC4_QM_GLBL_CFG1,
2148                        mmTPC4_QM_CP_STS,
2149                        mmTPC4_QM_GLBL_STS0);
2150
2151        if (rc) {
2152                dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
2153                retval = -EIO;
2154        }
2155
2156        rc = goya_stop_queue(hdev,
2157                        mmTPC4_CMDQ_GLBL_CFG1,
2158                        mmTPC4_CMDQ_CP_STS,
2159                        mmTPC4_CMDQ_GLBL_STS0);
2160
2161        if (rc) {
2162                dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
2163                retval = -EIO;
2164        }
2165
2166        rc = goya_stop_queue(hdev,
2167                        mmTPC5_QM_GLBL_CFG1,
2168                        mmTPC5_QM_CP_STS,
2169                        mmTPC5_QM_GLBL_STS0);
2170
2171        if (rc) {
2172                dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
2173                retval = -EIO;
2174        }
2175
2176        rc = goya_stop_queue(hdev,
2177                        mmTPC5_CMDQ_GLBL_CFG1,
2178                        mmTPC5_CMDQ_CP_STS,
2179                        mmTPC5_CMDQ_GLBL_STS0);
2180
2181        if (rc) {
2182                dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
2183                retval = -EIO;
2184        }
2185
2186        rc = goya_stop_queue(hdev,
2187                        mmTPC6_QM_GLBL_CFG1,
2188                        mmTPC6_QM_CP_STS,
2189                        mmTPC6_QM_GLBL_STS0);
2190
2191        if (rc) {
2192                dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
2193                retval = -EIO;
2194        }
2195
2196        rc = goya_stop_queue(hdev,
2197                        mmTPC6_CMDQ_GLBL_CFG1,
2198                        mmTPC6_CMDQ_CP_STS,
2199                        mmTPC6_CMDQ_GLBL_STS0);
2200
2201        if (rc) {
2202                dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
2203                retval = -EIO;
2204        }
2205
2206        rc = goya_stop_queue(hdev,
2207                        mmTPC7_QM_GLBL_CFG1,
2208                        mmTPC7_QM_CP_STS,
2209                        mmTPC7_QM_GLBL_STS0);
2210
2211        if (rc) {
2212                dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
2213                retval = -EIO;
2214        }
2215
2216        rc = goya_stop_queue(hdev,
2217                        mmTPC7_CMDQ_GLBL_CFG1,
2218                        mmTPC7_CMDQ_CP_STS,
2219                        mmTPC7_CMDQ_GLBL_STS0);
2220
2221        if (rc) {
2222                dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
2223                retval = -EIO;
2224        }
2225
2226        return retval;
2227}
2228
2229static void goya_dma_stall(struct hl_device *hdev)
2230{
2231        struct goya_device *goya = hdev->asic_specific;
2232
2233        if (!(goya->hw_cap_initialized & HW_CAP_DMA))
2234                return;
2235
2236        WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT);
2237        WREG32(mmDMA_QM_1_GLBL_CFG1, 1 << DMA_QM_1_GLBL_CFG1_DMA_STOP_SHIFT);
2238        WREG32(mmDMA_QM_2_GLBL_CFG1, 1 << DMA_QM_2_GLBL_CFG1_DMA_STOP_SHIFT);
2239        WREG32(mmDMA_QM_3_GLBL_CFG1, 1 << DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT);
2240        WREG32(mmDMA_QM_4_GLBL_CFG1, 1 << DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT);
2241}
2242
2243static void goya_tpc_stall(struct hl_device *hdev)
2244{
2245        struct goya_device *goya = hdev->asic_specific;
2246
2247        if (!(goya->hw_cap_initialized & HW_CAP_TPC))
2248                return;
2249
2250        WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
2251        WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC1_CFG_TPC_STALL_V_SHIFT);
2252        WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC2_CFG_TPC_STALL_V_SHIFT);
2253        WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC3_CFG_TPC_STALL_V_SHIFT);
2254        WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC4_CFG_TPC_STALL_V_SHIFT);
2255        WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC5_CFG_TPC_STALL_V_SHIFT);
2256        WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC6_CFG_TPC_STALL_V_SHIFT);
2257        WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC7_CFG_TPC_STALL_V_SHIFT);
2258}
2259
2260static void goya_mme_stall(struct hl_device *hdev)
2261{
2262        struct goya_device *goya = hdev->asic_specific;
2263
2264        if (!(goya->hw_cap_initialized & HW_CAP_MME))
2265                return;
2266
2267        WREG32(mmMME_STALL, 0xFFFFFFFF);
2268}
2269
2270static int goya_enable_msix(struct hl_device *hdev)
2271{
2272        struct goya_device *goya = hdev->asic_specific;
2273        int cq_cnt = hdev->asic_prop.completion_queues_count;
2274        int rc, i, irq_cnt_init, irq;
2275
2276        if (goya->hw_cap_initialized & HW_CAP_MSIX)
2277                return 0;
2278
2279        rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES,
2280                                GOYA_MSIX_ENTRIES, PCI_IRQ_MSIX);
2281        if (rc < 0) {
2282                dev_err(hdev->dev,
2283                        "MSI-X: Failed to enable support -- %d/%d\n",
2284                        GOYA_MSIX_ENTRIES, rc);
2285                return rc;
2286        }
2287
2288        for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
2289                irq = pci_irq_vector(hdev->pdev, i);
2290                rc = request_irq(irq, hl_irq_handler_cq, 0, goya_irq_name[i],
2291                                &hdev->completion_queue[i]);
2292                if (rc) {
2293                        dev_err(hdev->dev, "Failed to request IRQ %d", irq);
2294                        goto free_irqs;
2295                }
2296        }
2297
2298        irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2299
2300        rc = request_irq(irq, hl_irq_handler_eq, 0,
2301                        goya_irq_name[GOYA_EVENT_QUEUE_MSIX_IDX],
2302                        &hdev->event_queue);
2303        if (rc) {
2304                dev_err(hdev->dev, "Failed to request IRQ %d", irq);
2305                goto free_irqs;
2306        }
2307
2308        goya->hw_cap_initialized |= HW_CAP_MSIX;
2309        return 0;
2310
2311free_irqs:
2312        for (i = 0 ; i < irq_cnt_init ; i++)
2313                free_irq(pci_irq_vector(hdev->pdev, i),
2314                        &hdev->completion_queue[i]);
2315
2316        pci_free_irq_vectors(hdev->pdev);
2317        return rc;
2318}
2319
2320static void goya_sync_irqs(struct hl_device *hdev)
2321{
2322        struct goya_device *goya = hdev->asic_specific;
2323        int i;
2324
2325        if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
2326                return;
2327
2328        /* Wait for all pending IRQs to be finished */
2329        for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2330                synchronize_irq(pci_irq_vector(hdev->pdev, i));
2331
2332        synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX));
2333}
2334
2335static void goya_disable_msix(struct hl_device *hdev)
2336{
2337        struct goya_device *goya = hdev->asic_specific;
2338        int i, irq;
2339
2340        if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
2341                return;
2342
2343        goya_sync_irqs(hdev);
2344
2345        irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2346        free_irq(irq, &hdev->event_queue);
2347
2348        for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
2349                irq = pci_irq_vector(hdev->pdev, i);
2350                free_irq(irq, &hdev->completion_queue[i]);
2351        }
2352
2353        pci_free_irq_vectors(hdev->pdev);
2354
2355        goya->hw_cap_initialized &= ~HW_CAP_MSIX;
2356}
2357
2358static void goya_enable_timestamp(struct hl_device *hdev)
2359{
2360        /* Disable the timestamp counter */
2361        WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
2362
2363        /* Zero the lower/upper parts of the 64-bit counter */
2364        WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);
2365        WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);
2366
2367        /* Enable the counter */
2368        WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);
2369}
2370
2371static void goya_disable_timestamp(struct hl_device *hdev)
2372{
2373        /* Disable the timestamp counter */
2374        WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
2375}
2376
2377static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
2378{
2379        u32 wait_timeout_ms;
2380
2381        dev_info(hdev->dev,
2382                "Halting compute engines and disabling interrupts\n");
2383
2384        if (hdev->pldm)
2385                wait_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
2386        else
2387                wait_timeout_ms = GOYA_RESET_WAIT_MSEC;
2388
2389        goya_stop_external_queues(hdev);
2390        goya_stop_internal_queues(hdev);
2391
2392        msleep(wait_timeout_ms);
2393
2394        goya_dma_stall(hdev);
2395        goya_tpc_stall(hdev);
2396        goya_mme_stall(hdev);
2397
2398        msleep(wait_timeout_ms);
2399
2400        goya_disable_external_queues(hdev);
2401        goya_disable_internal_queues(hdev);
2402
2403        goya_disable_timestamp(hdev);
2404
2405        if (hard_reset) {
2406                goya_disable_msix(hdev);
2407                goya_mmu_remove_device_cpu_mappings(hdev);
2408        } else {
2409                goya_sync_irqs(hdev);
2410        }
2411}
2412
2413/*
2414 * goya_load_firmware_to_device() - Load LINUX FW code to device.
2415 * @hdev: Pointer to hl_device structure.
2416 *
2417 * Copy LINUX fw code from firmware file to HBM BAR.
2418 *
2419 * Return: 0 on success, non-zero for failure.
2420 */
2421static int goya_load_firmware_to_device(struct hl_device *hdev)
2422{
2423        void __iomem *dst;
2424
2425        dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
2426
2427        return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst, 0, 0);
2428}
2429
2430/*
2431 * goya_load_boot_fit_to_device() - Load boot fit to device.
2432 * @hdev: Pointer to hl_device structure.
2433 *
2434 * Copy boot fit file to SRAM BAR.
2435 *
2436 * Return: 0 on success, non-zero for failure.
2437 */
2438static int goya_load_boot_fit_to_device(struct hl_device *hdev)
2439{
2440        void __iomem *dst;
2441
2442        dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
2443
2444        return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst, 0, 0);
2445}
2446
2447static void goya_init_dynamic_firmware_loader(struct hl_device *hdev)
2448{
2449        struct dynamic_fw_load_mgr *dynamic_loader;
2450        struct cpu_dyn_regs *dyn_regs;
2451
2452        dynamic_loader = &hdev->fw_loader.dynamic_loader;
2453
2454        /*
2455         * here we update initial values for few specific dynamic regs (as
2456         * before reading the first descriptor from FW those value has to be
2457         * hard-coded) in later stages of the protocol those values will be
2458         * updated automatically by reading the FW descriptor so data there
2459         * will always be up-to-date
2460         */
2461        dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;
2462        dyn_regs->kmd_msg_to_cpu =
2463                                cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);
2464        dyn_regs->cpu_cmd_status_to_host =
2465                                cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);
2466
2467        dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC;
2468}
2469
2470static void goya_init_static_firmware_loader(struct hl_device *hdev)
2471{
2472        struct static_fw_load_mgr *static_loader;
2473
2474        static_loader = &hdev->fw_loader.static_loader;
2475
2476        static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;
2477        static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;
2478        static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU;
2479        static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST;
2480        static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;
2481        static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0;
2482        static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1;
2483        static_loader->boot_err0_reg = mmCPU_BOOT_ERR0;
2484        static_loader->boot_err1_reg = mmCPU_BOOT_ERR1;
2485        static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET;
2486        static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET;
2487        static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR));
2488}
2489
2490static void goya_init_firmware_loader(struct hl_device *hdev)
2491{
2492        struct asic_fixed_properties *prop = &hdev->asic_prop;
2493        struct fw_load_mgr *fw_loader = &hdev->fw_loader;
2494
2495        /* fill common fields */
2496        fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE;
2497        fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE;
2498        fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC;
2499        fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC;
2500        fw_loader->skip_bmc = false;
2501        fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;
2502        fw_loader->dram_bar_id = DDR_BAR_ID;
2503
2504        if (prop->dynamic_fw_load)
2505                goya_init_dynamic_firmware_loader(hdev);
2506        else
2507                goya_init_static_firmware_loader(hdev);
2508}
2509
2510static int goya_init_cpu(struct hl_device *hdev)
2511{
2512        struct goya_device *goya = hdev->asic_specific;
2513        int rc;
2514
2515        if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))
2516                return 0;
2517
2518        if (goya->hw_cap_initialized & HW_CAP_CPU)
2519                return 0;
2520
2521        /*
2522         * Before pushing u-boot/linux to device, need to set the ddr bar to
2523         * base address of dram
2524         */
2525        if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
2526                dev_err(hdev->dev,
2527                        "failed to map DDR bar to DRAM base address\n");
2528                return -EIO;
2529        }
2530
2531        rc = hl_fw_init_cpu(hdev);
2532
2533        if (rc)
2534                return rc;
2535
2536        goya->hw_cap_initialized |= HW_CAP_CPU;
2537
2538        return 0;
2539}
2540
2541static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
2542                                                u64 phys_addr)
2543{
2544        u32 status, timeout_usec;
2545        int rc;
2546
2547        if (hdev->pldm)
2548                timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
2549        else
2550                timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
2551
2552        WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
2553        WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
2554        WREG32(MMU_ASID_BUSY, 0x80000000 | asid);
2555
2556        rc = hl_poll_timeout(
2557                hdev,
2558                MMU_ASID_BUSY,
2559                status,
2560                !(status & 0x80000000),
2561                1000,
2562                timeout_usec);
2563
2564        if (rc) {
2565                dev_err(hdev->dev,
2566                        "Timeout during MMU hop0 config of asid %d\n", asid);
2567                return rc;
2568        }
2569
2570        return 0;
2571}
2572
2573int goya_mmu_init(struct hl_device *hdev)
2574{
2575        struct asic_fixed_properties *prop = &hdev->asic_prop;
2576        struct goya_device *goya = hdev->asic_specific;
2577        u64 hop0_addr;
2578        int rc, i;
2579
2580        if (!hdev->mmu_enable)
2581                return 0;
2582
2583        if (goya->hw_cap_initialized & HW_CAP_MMU)
2584                return 0;
2585
2586        hdev->dram_default_page_mapping = true;
2587
2588        for (i = 0 ; i < prop->max_asid ; i++) {
2589                hop0_addr = prop->mmu_pgt_addr +
2590                                (i * prop->mmu_hop_table_size);
2591
2592                rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
2593                if (rc) {
2594                        dev_err(hdev->dev,
2595                                "failed to set hop0 addr for asid %d\n", i);
2596                        goto err;
2597                }
2598        }
2599
2600        goya->hw_cap_initialized |= HW_CAP_MMU;
2601
2602        /* init MMU cache manage page */
2603        WREG32(mmSTLB_CACHE_INV_BASE_39_8,
2604                                lower_32_bits(MMU_CACHE_MNG_ADDR >> 8));
2605        WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
2606
2607        /* Remove follower feature due to performance bug */
2608        WREG32_AND(mmSTLB_STLB_FEATURE_EN,
2609                        (~STLB_STLB_FEATURE_EN_FOLLOWER_EN_MASK));
2610
2611        hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
2612                                        VM_TYPE_USERPTR | VM_TYPE_PHYS_PACK);
2613
2614        WREG32(mmMMU_MMU_ENABLE, 1);
2615        WREG32(mmMMU_SPI_MASK, 0xF);
2616
2617        return 0;
2618
2619err:
2620        return rc;
2621}
2622
2623/*
2624 * goya_hw_init - Goya hardware initialization code
2625 *
2626 * @hdev: pointer to hl_device structure
2627 *
2628 * Returns 0 on success
2629 *
2630 */
2631static int goya_hw_init(struct hl_device *hdev)
2632{
2633        struct asic_fixed_properties *prop = &hdev->asic_prop;
2634        int rc;
2635
2636        /* Perform read from the device to make sure device is up */
2637        RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
2638
2639        /*
2640         * Let's mark in the H/W that we have reached this point. We check
2641         * this value in the reset_before_init function to understand whether
2642         * we need to reset the chip before doing H/W init. This register is
2643         * cleared by the H/W upon H/W reset
2644         */
2645        WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
2646
2647        rc = goya_init_cpu(hdev);
2648        if (rc) {
2649                dev_err(hdev->dev, "failed to initialize CPU\n");
2650                return rc;
2651        }
2652
2653        goya_tpc_mbist_workaround(hdev);
2654
2655        goya_init_golden_registers(hdev);
2656
2657        /*
2658         * After CPU initialization is finished, change DDR bar mapping inside
2659         * iATU to point to the start address of the MMU page tables
2660         */
2661        if (goya_set_ddr_bar_base(hdev, (MMU_PAGE_TABLES_ADDR &
2662                        ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) {
2663                dev_err(hdev->dev,
2664                        "failed to map DDR bar to MMU page tables\n");
2665                return -EIO;
2666        }
2667
2668        rc = goya_mmu_init(hdev);
2669        if (rc)
2670                return rc;
2671
2672        goya_init_security(hdev);
2673
2674        goya_init_dma_qmans(hdev);
2675
2676        goya_init_mme_qmans(hdev);
2677
2678        goya_init_tpc_qmans(hdev);
2679
2680        goya_enable_timestamp(hdev);
2681
2682        /* MSI-X must be enabled before CPU queues are initialized */
2683        rc = goya_enable_msix(hdev);
2684        if (rc)
2685                goto disable_queues;
2686
2687        /* Perform read from the device to flush all MSI-X configuration */
2688        RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
2689
2690        return 0;
2691
2692disable_queues:
2693        goya_disable_internal_queues(hdev);
2694        goya_disable_external_queues(hdev);
2695
2696        return rc;
2697}
2698
2699/*
2700 * goya_hw_fini - Goya hardware tear-down code
2701 *
2702 * @hdev: pointer to hl_device structure
2703 * @hard_reset: should we do hard reset to all engines or just reset the
2704 *              compute/dma engines
2705 */
2706static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
2707{
2708        struct goya_device *goya = hdev->asic_specific;
2709        u32 reset_timeout_ms, cpu_timeout_ms, status;
2710
2711        if (hdev->pldm) {
2712                reset_timeout_ms = GOYA_PLDM_RESET_TIMEOUT_MSEC;
2713                cpu_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
2714        } else {
2715                reset_timeout_ms = GOYA_RESET_TIMEOUT_MSEC;
2716                cpu_timeout_ms = GOYA_CPU_RESET_WAIT_MSEC;
2717        }
2718
2719        if (hard_reset) {
2720                /* I don't know what is the state of the CPU so make sure it is
2721                 * stopped in any means necessary
2722                 */
2723                WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_GOTO_WFE);
2724                WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2725                        GOYA_ASYNC_EVENT_ID_HALT_MACHINE);
2726
2727                msleep(cpu_timeout_ms);
2728
2729                goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
2730                goya_disable_clk_rlx(hdev);
2731                goya_set_pll_refclk(hdev);
2732
2733                WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, RESET_ALL);
2734                dev_info(hdev->dev,
2735                        "Issued HARD reset command, going to wait %dms\n",
2736                        reset_timeout_ms);
2737        } else {
2738                WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, DMA_MME_TPC_RESET);
2739                dev_info(hdev->dev,
2740                        "Issued SOFT reset command, going to wait %dms\n",
2741                        reset_timeout_ms);
2742        }
2743
2744        /*
2745         * After hard reset, we can't poll the BTM_FSM register because the PSOC
2746         * itself is in reset. In either reset we need to wait until the reset
2747         * is deasserted
2748         */
2749        msleep(reset_timeout_ms);
2750
2751        status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
2752        if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
2753                dev_err(hdev->dev,
2754                        "Timeout while waiting for device to reset 0x%x\n",
2755                        status);
2756
2757        if (!hard_reset && goya) {
2758                goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME |
2759                                                HW_CAP_GOLDEN | HW_CAP_TPC);
2760                WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2761                                GOYA_ASYNC_EVENT_ID_SOFT_RESET);
2762                return;
2763        }
2764
2765        /* Chicken bit to re-initiate boot sequencer flow */
2766        WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START,
2767                1 << PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_SHIFT);
2768        /* Move boot manager FSM to pre boot sequencer init state */
2769        WREG32(mmPSOC_GLOBAL_CONF_SW_BTM_FSM,
2770                        0xA << PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_SHIFT);
2771
2772        if (goya) {
2773                goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |
2774                                HW_CAP_DDR_0 | HW_CAP_DDR_1 |
2775                                HW_CAP_DMA | HW_CAP_MME |
2776                                HW_CAP_MMU | HW_CAP_TPC_MBIST |
2777                                HW_CAP_GOLDEN | HW_CAP_TPC);
2778
2779                memset(goya->events_stat, 0, sizeof(goya->events_stat));
2780        }
2781}
2782
2783int goya_suspend(struct hl_device *hdev)
2784{
2785        int rc;
2786
2787        rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
2788        if (rc)
2789                dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
2790
2791        return rc;
2792}
2793
2794int goya_resume(struct hl_device *hdev)
2795{
2796        return goya_init_iatu(hdev);
2797}
2798
2799static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
2800                        void *cpu_addr, dma_addr_t dma_addr, size_t size)
2801{
2802        int rc;
2803
2804        vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
2805                        VM_DONTCOPY | VM_NORESERVE;
2806
2807        rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr,
2808                                (dma_addr - HOST_PHYS_BASE), size);
2809        if (rc)
2810                dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
2811
2812        return rc;
2813}
2814
2815void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
2816{
2817        u32 db_reg_offset, db_value;
2818
2819        switch (hw_queue_id) {
2820        case GOYA_QUEUE_ID_DMA_0:
2821                db_reg_offset = mmDMA_QM_0_PQ_PI;
2822                break;
2823
2824        case GOYA_QUEUE_ID_DMA_1:
2825                db_reg_offset = mmDMA_QM_1_PQ_PI;
2826                break;
2827
2828        case GOYA_QUEUE_ID_DMA_2:
2829                db_reg_offset = mmDMA_QM_2_PQ_PI;
2830                break;
2831
2832        case GOYA_QUEUE_ID_DMA_3:
2833                db_reg_offset = mmDMA_QM_3_PQ_PI;
2834                break;
2835
2836        case GOYA_QUEUE_ID_DMA_4:
2837                db_reg_offset = mmDMA_QM_4_PQ_PI;
2838                break;
2839
2840        case GOYA_QUEUE_ID_CPU_PQ:
2841                db_reg_offset = mmCPU_IF_PF_PQ_PI;
2842                break;
2843
2844        case GOYA_QUEUE_ID_MME:
2845                db_reg_offset = mmMME_QM_PQ_PI;
2846                break;
2847
2848        case GOYA_QUEUE_ID_TPC0:
2849                db_reg_offset = mmTPC0_QM_PQ_PI;
2850                break;
2851
2852        case GOYA_QUEUE_ID_TPC1:
2853                db_reg_offset = mmTPC1_QM_PQ_PI;
2854                break;
2855
2856        case GOYA_QUEUE_ID_TPC2:
2857                db_reg_offset = mmTPC2_QM_PQ_PI;
2858                break;
2859
2860        case GOYA_QUEUE_ID_TPC3:
2861                db_reg_offset = mmTPC3_QM_PQ_PI;
2862                break;
2863
2864        case GOYA_QUEUE_ID_TPC4:
2865                db_reg_offset = mmTPC4_QM_PQ_PI;
2866                break;
2867
2868        case GOYA_QUEUE_ID_TPC5:
2869                db_reg_offset = mmTPC5_QM_PQ_PI;
2870                break;
2871
2872        case GOYA_QUEUE_ID_TPC6:
2873                db_reg_offset = mmTPC6_QM_PQ_PI;
2874                break;
2875
2876        case GOYA_QUEUE_ID_TPC7:
2877                db_reg_offset = mmTPC7_QM_PQ_PI;
2878                break;
2879
2880        default:
2881                /* Should never get here */
2882                dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n",
2883                        hw_queue_id);
2884                return;
2885        }
2886
2887        db_value = pi;
2888
2889        /* ring the doorbell */
2890        WREG32(db_reg_offset, db_value);
2891
2892        if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ) {
2893                /* make sure device CPU will read latest data from host */
2894                mb();
2895                WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2896                                GOYA_ASYNC_EVENT_ID_PI_UPDATE);
2897        }
2898}
2899
2900void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
2901{
2902        /* The QMANs are on the SRAM so need to copy to IO space */
2903        memcpy_toio((void __iomem *) pqe, bd, sizeof(struct hl_bd));
2904}
2905
2906static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size,
2907                                        dma_addr_t *dma_handle, gfp_t flags)
2908{
2909        void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
2910                                                dma_handle, flags);
2911
2912        /* Shift to the device's base physical address of host memory */
2913        if (kernel_addr)
2914                *dma_handle += HOST_PHYS_BASE;
2915
2916        return kernel_addr;
2917}
2918
2919static void goya_dma_free_coherent(struct hl_device *hdev, size_t size,
2920                                        void *cpu_addr, dma_addr_t dma_handle)
2921{
2922        /* Cancel the device's base physical address of host memory */
2923        dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
2924
2925        dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
2926}
2927
2928int goya_scrub_device_mem(struct hl_device *hdev, u64 addr, u64 size)
2929{
2930        return 0;
2931}
2932
2933void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
2934                                dma_addr_t *dma_handle, u16 *queue_len)
2935{
2936        void *base;
2937        u32 offset;
2938
2939        *dma_handle = hdev->asic_prop.sram_base_address;
2940
2941        base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
2942
2943        switch (queue_id) {
2944        case GOYA_QUEUE_ID_MME:
2945                offset = MME_QMAN_BASE_OFFSET;
2946                *queue_len = MME_QMAN_LENGTH;
2947                break;
2948        case GOYA_QUEUE_ID_TPC0:
2949                offset = TPC0_QMAN_BASE_OFFSET;
2950                *queue_len = TPC_QMAN_LENGTH;
2951                break;
2952        case GOYA_QUEUE_ID_TPC1:
2953                offset = TPC1_QMAN_BASE_OFFSET;
2954                *queue_len = TPC_QMAN_LENGTH;
2955                break;
2956        case GOYA_QUEUE_ID_TPC2:
2957                offset = TPC2_QMAN_BASE_OFFSET;
2958                *queue_len = TPC_QMAN_LENGTH;
2959                break;
2960        case GOYA_QUEUE_ID_TPC3:
2961                offset = TPC3_QMAN_BASE_OFFSET;
2962                *queue_len = TPC_QMAN_LENGTH;
2963                break;
2964        case GOYA_QUEUE_ID_TPC4:
2965                offset = TPC4_QMAN_BASE_OFFSET;
2966                *queue_len = TPC_QMAN_LENGTH;
2967                break;
2968        case GOYA_QUEUE_ID_TPC5:
2969                offset = TPC5_QMAN_BASE_OFFSET;
2970                *queue_len = TPC_QMAN_LENGTH;
2971                break;
2972        case GOYA_QUEUE_ID_TPC6:
2973                offset = TPC6_QMAN_BASE_OFFSET;
2974                *queue_len = TPC_QMAN_LENGTH;
2975                break;
2976        case GOYA_QUEUE_ID_TPC7:
2977                offset = TPC7_QMAN_BASE_OFFSET;
2978                *queue_len = TPC_QMAN_LENGTH;
2979                break;
2980        default:
2981                dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
2982                return NULL;
2983        }
2984
2985        base += offset;
2986        *dma_handle += offset;
2987
2988        return base;
2989}
2990
2991static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
2992{
2993        struct packet_msg_prot *fence_pkt;
2994        u32 *fence_ptr;
2995        dma_addr_t fence_dma_addr;
2996        struct hl_cb *cb;
2997        u32 tmp, timeout;
2998        int rc;
2999
3000        if (hdev->pldm)
3001                timeout = GOYA_PLDM_QMAN0_TIMEOUT_USEC;
3002        else
3003                timeout = HL_DEVICE_TIMEOUT_USEC;
3004
3005        if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) {
3006                dev_err_ratelimited(hdev->dev,
3007                        "Can't send driver job on QMAN0 because the device is not idle\n");
3008                return -EBUSY;
3009        }
3010
3011        fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
3012                                                        &fence_dma_addr);
3013        if (!fence_ptr) {
3014                dev_err(hdev->dev,
3015                        "Failed to allocate fence memory for QMAN0\n");
3016                return -ENOMEM;
3017        }
3018
3019        goya_qman0_set_security(hdev, true);
3020
3021        cb = job->patched_cb;
3022
3023        fence_pkt = cb->kernel_address +
3024                        job->job_cb_size - sizeof(struct packet_msg_prot);
3025
3026        tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
3027                        (1 << GOYA_PKT_CTL_EB_SHIFT) |
3028                        (1 << GOYA_PKT_CTL_MB_SHIFT);
3029        fence_pkt->ctl = cpu_to_le32(tmp);
3030        fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL);
3031        fence_pkt->addr = cpu_to_le64(fence_dma_addr);
3032
3033        rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0,
3034                                        job->job_cb_size, cb->bus_address);
3035        if (rc) {
3036                dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
3037                goto free_fence_ptr;
3038        }
3039
3040        rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
3041                                (tmp == GOYA_QMAN0_FENCE_VAL), 1000,
3042                                timeout, true);
3043
3044        hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0);
3045
3046        if (rc == -ETIMEDOUT) {
3047                dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
3048                goto free_fence_ptr;
3049        }
3050
3051free_fence_ptr:
3052        hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
3053                                        fence_dma_addr);
3054
3055        goya_qman0_set_security(hdev, false);
3056
3057        return rc;
3058}
3059
3060int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
3061                                u32 timeout, u64 *result)
3062{
3063        struct goya_device *goya = hdev->asic_specific;
3064
3065        if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
3066                if (result)
3067                        *result = 0;
3068                return 0;
3069        }
3070
3071        if (!timeout)
3072                timeout = GOYA_MSG_TO_CPU_TIMEOUT_USEC;
3073
3074        return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len,
3075                                        timeout, result);
3076}
3077
3078int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
3079{
3080        struct packet_msg_prot *fence_pkt;
3081        dma_addr_t pkt_dma_addr;
3082        u32 fence_val, tmp;
3083        dma_addr_t fence_dma_addr;
3084        u32 *fence_ptr;
3085        int rc;
3086
3087        fence_val = GOYA_QMAN0_FENCE_VAL;
3088
3089        fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
3090                                                        &fence_dma_addr);
3091        if (!fence_ptr) {
3092                dev_err(hdev->dev,
3093                        "Failed to allocate memory for H/W queue %d testing\n",
3094                        hw_queue_id);
3095                return -ENOMEM;
3096        }
3097
3098        *fence_ptr = 0;
3099
3100        fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
3101                                        sizeof(struct packet_msg_prot),
3102                                        GFP_KERNEL, &pkt_dma_addr);
3103        if (!fence_pkt) {
3104                dev_err(hdev->dev,
3105                        "Failed to allocate packet for H/W queue %d testing\n",
3106                        hw_queue_id);
3107                rc = -ENOMEM;
3108                goto free_fence_ptr;
3109        }
3110
3111        tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
3112                        (1 << GOYA_PKT_CTL_EB_SHIFT) |
3113                        (1 << GOYA_PKT_CTL_MB_SHIFT);
3114        fence_pkt->ctl = cpu_to_le32(tmp);
3115        fence_pkt->value = cpu_to_le32(fence_val);
3116        fence_pkt->addr = cpu_to_le64(fence_dma_addr);
3117
3118        rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
3119                                        sizeof(struct packet_msg_prot),
3120                                        pkt_dma_addr);
3121        if (rc) {
3122                dev_err(hdev->dev,
3123                        "Failed to send fence packet to H/W queue %d\n",
3124                        hw_queue_id);
3125                goto free_pkt;
3126        }
3127
3128        rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
3129                                        1000, GOYA_TEST_QUEUE_WAIT_USEC, true);
3130
3131        hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
3132
3133        if (rc == -ETIMEDOUT) {
3134                dev_err(hdev->dev,
3135                        "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
3136                        hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
3137                rc = -EIO;
3138        }
3139
3140free_pkt:
3141        hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
3142                                        pkt_dma_addr);
3143free_fence_ptr:
3144        hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
3145                                        fence_dma_addr);
3146        return rc;
3147}
3148
3149int goya_test_cpu_queue(struct hl_device *hdev)
3150{
3151        struct goya_device *goya = hdev->asic_specific;
3152
3153        /*
3154         * check capability here as send_cpu_message() won't update the result
3155         * value if no capability
3156         */
3157        if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
3158                return 0;
3159
3160        return hl_fw_test_cpu_queue(hdev);
3161}
3162
3163int goya_test_queues(struct hl_device *hdev)
3164{
3165        int i, rc, ret_val = 0;
3166
3167        for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
3168                rc = goya_test_queue(hdev, i);
3169                if (rc)
3170                        ret_val = -EINVAL;
3171        }
3172
3173        return ret_val;
3174}
3175
3176static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size,
3177                                        gfp_t mem_flags, dma_addr_t *dma_handle)
3178{
3179        void *kernel_addr;
3180
3181        if (size > GOYA_DMA_POOL_BLK_SIZE)
3182                return NULL;
3183
3184        kernel_addr =  dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
3185
3186        /* Shift to the device's base physical address of host memory */
3187        if (kernel_addr)
3188                *dma_handle += HOST_PHYS_BASE;
3189
3190        return kernel_addr;
3191}
3192
3193static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr,
3194                                dma_addr_t dma_addr)
3195{
3196        /* Cancel the device's base physical address of host memory */
3197        dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
3198
3199        dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
3200}
3201
3202void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
3203                                        dma_addr_t *dma_handle)
3204{
3205        void *vaddr;
3206
3207        vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
3208        *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address +
3209                        VA_CPU_ACCESSIBLE_MEM_ADDR;
3210
3211        return vaddr;
3212}
3213
3214void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
3215                                        void *vaddr)
3216{
3217        hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
3218}
3219
3220static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
3221                                int nents, enum dma_data_direction dir)
3222{
3223        struct scatterlist *sg;
3224        int i;
3225
3226        if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
3227                return -ENOMEM;
3228
3229        /* Shift to the device's base physical address of host memory */
3230        for_each_sg(sgl, sg, nents, i)
3231                sg->dma_address += HOST_PHYS_BASE;
3232
3233        return 0;
3234}
3235
3236static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
3237                                int nents, enum dma_data_direction dir)
3238{
3239        struct scatterlist *sg;
3240        int i;
3241
3242        /* Cancel the device's base physical address of host memory */
3243        for_each_sg(sgl, sg, nents, i)
3244                sg->dma_address -= HOST_PHYS_BASE;
3245
3246        dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
3247}
3248
3249u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
3250{
3251        struct scatterlist *sg, *sg_next_iter;
3252        u32 count, dma_desc_cnt;
3253        u64 len, len_next;
3254        dma_addr_t addr, addr_next;
3255
3256        dma_desc_cnt = 0;
3257
3258        for_each_sg(sgt->sgl, sg, sgt->nents, count) {
3259
3260                len = sg_dma_len(sg);
3261                addr = sg_dma_address(sg);
3262
3263                if (len == 0)
3264                        break;
3265
3266                while ((count + 1) < sgt->nents) {
3267                        sg_next_iter = sg_next(sg);
3268                        len_next = sg_dma_len(sg_next_iter);
3269                        addr_next = sg_dma_address(sg_next_iter);
3270
3271                        if (len_next == 0)
3272                                break;
3273
3274                        if ((addr + len == addr_next) &&
3275                                (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
3276                                len += len_next;
3277                                count++;
3278                                sg = sg_next_iter;
3279                        } else {
3280                                break;
3281                        }
3282                }
3283
3284                dma_desc_cnt++;
3285        }
3286
3287        return dma_desc_cnt * sizeof(struct packet_lin_dma);
3288}
3289
3290static int goya_pin_memory_before_cs(struct hl_device *hdev,
3291                                struct hl_cs_parser *parser,
3292                                struct packet_lin_dma *user_dma_pkt,
3293                                u64 addr, enum dma_data_direction dir)
3294{
3295        struct hl_userptr *userptr;
3296        int rc;
3297
3298        if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3299                        parser->job_userptr_list, &userptr))
3300                goto already_pinned;
3301
3302        userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);
3303        if (!userptr)
3304                return -ENOMEM;
3305
3306        rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3307                                userptr);
3308        if (rc)
3309                goto free_userptr;
3310
3311        list_add_tail(&userptr->job_node, parser->job_userptr_list);
3312
3313        rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
3314                                        userptr->sgt->nents, dir);
3315        if (rc) {
3316                dev_err(hdev->dev, "failed to map sgt with DMA region\n");
3317                goto unpin_memory;
3318        }
3319
3320        userptr->dma_mapped = true;
3321        userptr->dir = dir;
3322
3323already_pinned:
3324        parser->patched_cb_size +=
3325                        goya_get_dma_desc_list_size(hdev, userptr->sgt);
3326
3327        return 0;
3328
3329unpin_memory:
3330        list_del(&userptr->job_node);
3331        hl_unpin_host_memory(hdev, userptr);
3332free_userptr:
3333        kfree(userptr);
3334        return rc;
3335}
3336
3337static int goya_validate_dma_pkt_host(struct hl_device *hdev,
3338                                struct hl_cs_parser *parser,
3339                                struct packet_lin_dma *user_dma_pkt)
3340{
3341        u64 device_memory_addr, addr;
3342        enum dma_data_direction dir;
3343        enum goya_dma_direction user_dir;
3344        bool sram_addr = true;
3345        bool skip_host_mem_pin = false;
3346        bool user_memset;
3347        u32 ctl;
3348        int rc = 0;
3349
3350        ctl = le32_to_cpu(user_dma_pkt->ctl);
3351
3352        user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3353                        GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3354
3355        user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
3356                        GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
3357
3358        switch (user_dir) {
3359        case DMA_HOST_TO_DRAM:
3360                dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n");
3361                dir = DMA_TO_DEVICE;
3362                sram_addr = false;
3363                addr = le64_to_cpu(user_dma_pkt->src_addr);
3364                device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3365                if (user_memset)
3366                        skip_host_mem_pin = true;
3367                break;
3368
3369        case DMA_DRAM_TO_HOST:
3370                dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n");
3371                dir = DMA_FROM_DEVICE;
3372                sram_addr = false;
3373                addr = le64_to_cpu(user_dma_pkt->dst_addr);
3374                device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3375                break;
3376
3377        case DMA_HOST_TO_SRAM:
3378                dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n");
3379                dir = DMA_TO_DEVICE;
3380                addr = le64_to_cpu(user_dma_pkt->src_addr);
3381                device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3382                if (user_memset)
3383                        skip_host_mem_pin = true;
3384                break;
3385
3386        case DMA_SRAM_TO_HOST:
3387                dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n");
3388                dir = DMA_FROM_DEVICE;
3389                addr = le64_to_cpu(user_dma_pkt->dst_addr);
3390                device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3391                break;
3392        default:
3393                dev_err(hdev->dev, "DMA direction is undefined\n");
3394                return -EFAULT;
3395        }
3396
3397        if (sram_addr) {
3398                if (!hl_mem_area_inside_range(device_memory_addr,
3399                                le32_to_cpu(user_dma_pkt->tsize),
3400                                hdev->asic_prop.sram_user_base_address,
3401                                hdev->asic_prop.sram_end_address)) {
3402
3403                        dev_err(hdev->dev,
3404                                "SRAM address 0x%llx + 0x%x is invalid\n",
3405                                device_memory_addr,
3406                                user_dma_pkt->tsize);
3407                        return -EFAULT;
3408                }
3409        } else {
3410                if (!hl_mem_area_inside_range(device_memory_addr,
3411                                le32_to_cpu(user_dma_pkt->tsize),
3412                                hdev->asic_prop.dram_user_base_address,
3413                                hdev->asic_prop.dram_end_address)) {
3414
3415                        dev_err(hdev->dev,
3416                                "DRAM address 0x%llx + 0x%x is invalid\n",
3417                                device_memory_addr,
3418                                user_dma_pkt->tsize);
3419                        return -EFAULT;
3420                }
3421        }
3422
3423        if (skip_host_mem_pin)
3424                parser->patched_cb_size += sizeof(*user_dma_pkt);
3425        else {
3426                if ((dir == DMA_TO_DEVICE) &&
3427                                (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) {
3428                        dev_err(hdev->dev,
3429                                "Can't DMA from host on queue other then 1\n");
3430                        return -EFAULT;
3431                }
3432
3433                rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt,
3434                                                addr, dir);
3435        }
3436
3437        return rc;
3438}
3439
3440static int goya_validate_dma_pkt_no_host(struct hl_device *hdev,
3441                                struct hl_cs_parser *parser,
3442                                struct packet_lin_dma *user_dma_pkt)
3443{
3444        u64 sram_memory_addr, dram_memory_addr;
3445        enum goya_dma_direction user_dir;
3446        u32 ctl;
3447
3448        ctl = le32_to_cpu(user_dma_pkt->ctl);
3449        user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3450                        GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3451
3452        if (user_dir == DMA_DRAM_TO_SRAM) {
3453                dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n");
3454                dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3455                sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3456        } else {
3457                dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n");
3458                sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3459                dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3460        }
3461
3462        if (!hl_mem_area_inside_range(sram_memory_addr,
3463                                le32_to_cpu(user_dma_pkt->tsize),
3464                                hdev->asic_prop.sram_user_base_address,
3465                                hdev->asic_prop.sram_end_address)) {
3466                dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n",
3467                        sram_memory_addr, user_dma_pkt->tsize);
3468                return -EFAULT;
3469        }
3470
3471        if (!hl_mem_area_inside_range(dram_memory_addr,
3472                                le32_to_cpu(user_dma_pkt->tsize),
3473                                hdev->asic_prop.dram_user_base_address,
3474                                hdev->asic_prop.dram_end_address)) {
3475                dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n",
3476                        dram_memory_addr, user_dma_pkt->tsize);
3477                return -EFAULT;
3478        }
3479
3480        parser->patched_cb_size += sizeof(*user_dma_pkt);
3481
3482        return 0;
3483}
3484
3485static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev,
3486                                struct hl_cs_parser *parser,
3487                                struct packet_lin_dma *user_dma_pkt)
3488{
3489        enum goya_dma_direction user_dir;
3490        u32 ctl;
3491        int rc;
3492
3493        dev_dbg(hdev->dev, "DMA packet details:\n");
3494        dev_dbg(hdev->dev, "source == 0x%llx\n",
3495                le64_to_cpu(user_dma_pkt->src_addr));
3496        dev_dbg(hdev->dev, "destination == 0x%llx\n",
3497                le64_to_cpu(user_dma_pkt->dst_addr));
3498        dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3499
3500        ctl = le32_to_cpu(user_dma_pkt->ctl);
3501        user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3502                        GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3503
3504        /*
3505         * Special handling for DMA with size 0. The H/W has a bug where
3506         * this can cause the QMAN DMA to get stuck, so block it here.
3507         */
3508        if (user_dma_pkt->tsize == 0) {
3509                dev_err(hdev->dev,
3510                        "Got DMA with size 0, might reset the device\n");
3511                return -EINVAL;
3512        }
3513
3514        if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM))
3515                rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt);
3516        else
3517                rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt);
3518
3519        return rc;
3520}
3521
3522static int goya_validate_dma_pkt_mmu(struct hl_device *hdev,
3523                                struct hl_cs_parser *parser,
3524                                struct packet_lin_dma *user_dma_pkt)
3525{
3526        dev_dbg(hdev->dev, "DMA packet details:\n");
3527        dev_dbg(hdev->dev, "source == 0x%llx\n",
3528                le64_to_cpu(user_dma_pkt->src_addr));
3529        dev_dbg(hdev->dev, "destination == 0x%llx\n",
3530                le64_to_cpu(user_dma_pkt->dst_addr));
3531        dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3532
3533        /*
3534         * WA for HW-23.
3535         * We can't allow user to read from Host using QMANs other than 1.
3536         * PMMU and HPMMU addresses are equal, check only one of them.
3537         */
3538        if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 &&
3539                hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr),
3540                                le32_to_cpu(user_dma_pkt->tsize),
3541                                hdev->asic_prop.pmmu.start_addr,
3542                                hdev->asic_prop.pmmu.end_addr)) {
3543                dev_err(hdev->dev,
3544                        "Can't DMA from host on queue other then 1\n");
3545                return -EFAULT;
3546        }
3547
3548        if (user_dma_pkt->tsize == 0) {
3549                dev_err(hdev->dev,
3550                        "Got DMA with size 0, might reset the device\n");
3551                return -EINVAL;
3552        }
3553
3554        parser->patched_cb_size += sizeof(*user_dma_pkt);
3555
3556        return 0;
3557}
3558
3559static int goya_validate_wreg32(struct hl_device *hdev,
3560                                struct hl_cs_parser *parser,
3561                                struct packet_wreg32 *wreg_pkt)
3562{
3563        struct goya_device *goya = hdev->asic_specific;
3564        u32 sob_start_addr, sob_end_addr;
3565        u16 reg_offset;
3566
3567        reg_offset = le32_to_cpu(wreg_pkt->ctl) &
3568                        GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK;
3569
3570        dev_dbg(hdev->dev, "WREG32 packet details:\n");
3571        dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset);
3572        dev_dbg(hdev->dev, "value      == 0x%x\n",
3573                le32_to_cpu(wreg_pkt->value));
3574
3575        if (reg_offset != (mmDMA_CH_0_WR_COMP_ADDR_LO & 0x1FFF)) {
3576                dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n",
3577                        reg_offset);
3578                return -EPERM;
3579        }
3580
3581        /*
3582         * With MMU, DMA channels are not secured, so it doesn't matter where
3583         * the WR COMP will be written to because it will go out with
3584         * non-secured property
3585         */
3586        if (goya->hw_cap_initialized & HW_CAP_MMU)
3587                return 0;
3588
3589        sob_start_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
3590        sob_end_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1023);
3591
3592        if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) ||
3593                        (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) {
3594
3595                dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n",
3596                        wreg_pkt->value);
3597                return -EPERM;
3598        }
3599
3600        return 0;
3601}
3602
3603static int goya_validate_cb(struct hl_device *hdev,
3604                        struct hl_cs_parser *parser, bool is_mmu)
3605{
3606        u32 cb_parsed_length = 0;
3607        int rc = 0;
3608
3609        parser->patched_cb_size = 0;
3610
3611        /* cb_user_size is more than 0 so loop will always be executed */
3612        while (cb_parsed_length < parser->user_cb_size) {
3613                enum packet_id pkt_id;
3614                u16 pkt_size;
3615                struct goya_packet *user_pkt;
3616
3617                user_pkt = parser->user_cb->kernel_address + cb_parsed_length;
3618
3619                pkt_id = (enum packet_id) (
3620                                (le64_to_cpu(user_pkt->header) &
3621                                PACKET_HEADER_PACKET_ID_MASK) >>
3622                                        PACKET_HEADER_PACKET_ID_SHIFT);
3623
3624                if (!validate_packet_id(pkt_id)) {
3625                        dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
3626                        rc = -EINVAL;
3627                        break;
3628                }
3629
3630                pkt_size = goya_packet_sizes[pkt_id];
3631                cb_parsed_length += pkt_size;
3632                if (cb_parsed_length > parser->user_cb_size) {
3633                        dev_err(hdev->dev,
3634                                "packet 0x%x is out of CB boundary\n", pkt_id);
3635                        rc = -EINVAL;
3636                        break;
3637                }
3638
3639                switch (pkt_id) {
3640                case PACKET_WREG_32:
3641                        /*
3642                         * Although it is validated after copy in patch_cb(),
3643                         * need to validate here as well because patch_cb() is
3644                         * not called in MMU path while this function is called
3645                         */
3646                        rc = goya_validate_wreg32(hdev,
3647                                parser, (struct packet_wreg32 *) user_pkt);
3648                        parser->patched_cb_size += pkt_size;
3649                        break;
3650
3651                case PACKET_WREG_BULK:
3652                        dev_err(hdev->dev,
3653                                "User not allowed to use WREG_BULK\n");
3654                        rc = -EPERM;
3655                        break;
3656
3657                case PACKET_MSG_PROT:
3658                        dev_err(hdev->dev,
3659                                "User not allowed to use MSG_PROT\n");
3660                        rc = -EPERM;
3661                        break;
3662
3663                case PACKET_CP_DMA:
3664                        dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3665                        rc = -EPERM;
3666                        break;
3667
3668                case PACKET_STOP:
3669                        dev_err(hdev->dev, "User not allowed to use STOP\n");
3670                        rc = -EPERM;
3671                        break;
3672
3673                case PACKET_LIN_DMA:
3674                        if (is_mmu)
3675                                rc = goya_validate_dma_pkt_mmu(hdev, parser,
3676                                        (struct packet_lin_dma *) user_pkt);
3677                        else
3678                                rc = goya_validate_dma_pkt_no_mmu(hdev, parser,
3679                                        (struct packet_lin_dma *) user_pkt);
3680                        break;
3681
3682                case PACKET_MSG_LONG:
3683                case PACKET_MSG_SHORT:
3684                case PACKET_FENCE:
3685                case PACKET_NOP:
3686                        parser->patched_cb_size += pkt_size;
3687                        break;
3688
3689                default:
3690                        dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3691                                pkt_id);
3692                        rc = -EINVAL;
3693                        break;
3694                }
3695
3696                if (rc)
3697                        break;
3698        }
3699
3700        /*
3701         * The new CB should have space at the end for two MSG_PROT packets:
3702         * 1. A packet that will act as a completion packet
3703         * 2. A packet that will generate MSI-X interrupt
3704         */
3705        parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2;
3706
3707        return rc;
3708}
3709
3710static int goya_patch_dma_packet(struct hl_device *hdev,
3711                                struct hl_cs_parser *parser,
3712                                struct packet_lin_dma *user_dma_pkt,
3713                                struct packet_lin_dma *new_dma_pkt,
3714                                u32 *new_dma_pkt_size)
3715{
3716        struct hl_userptr *userptr;
3717        struct scatterlist *sg, *sg_next_iter;
3718        u32 count, dma_desc_cnt;
3719        u64 len, len_next;
3720        dma_addr_t dma_addr, dma_addr_next;
3721        enum goya_dma_direction user_dir;
3722        u64 device_memory_addr, addr;
3723        enum dma_data_direction dir;
3724        struct sg_table *sgt;
3725        bool skip_host_mem_pin = false;
3726        bool user_memset;
3727        u32 user_rdcomp_mask, user_wrcomp_mask, ctl;
3728
3729        ctl = le32_to_cpu(user_dma_pkt->ctl);
3730
3731        user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3732                        GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3733
3734        user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
3735                        GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
3736
3737        if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM) ||
3738                        (user_dma_pkt->tsize == 0)) {
3739                memcpy(new_dma_pkt, user_dma_pkt, sizeof(*new_dma_pkt));
3740                *new_dma_pkt_size = sizeof(*new_dma_pkt);
3741                return 0;
3742        }
3743
3744        if ((user_dir == DMA_HOST_TO_DRAM) || (user_dir == DMA_HOST_TO_SRAM)) {
3745                addr = le64_to_cpu(user_dma_pkt->src_addr);
3746                device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3747                dir = DMA_TO_DEVICE;
3748                if (user_memset)
3749                        skip_host_mem_pin = true;
3750        } else {
3751                addr = le64_to_cpu(user_dma_pkt->dst_addr);
3752                device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3753                dir = DMA_FROM_DEVICE;
3754        }
3755
3756        if ((!skip_host_mem_pin) &&
3757                (hl_userptr_is_pinned(hdev, addr,
3758                        le32_to_cpu(user_dma_pkt->tsize),
3759                        parser->job_userptr_list, &userptr) == false)) {
3760                dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
3761                                addr, user_dma_pkt->tsize);
3762                return -EFAULT;
3763        }
3764
3765        if ((user_memset) && (dir == DMA_TO_DEVICE)) {
3766                memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));
3767                *new_dma_pkt_size = sizeof(*user_dma_pkt);
3768                return 0;
3769        }
3770
3771        user_rdcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK;
3772
3773        user_wrcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK;
3774
3775        sgt = userptr->sgt;
3776        dma_desc_cnt = 0;
3777
3778        for_each_sg(sgt->sgl, sg, sgt->nents, count) {
3779                len = sg_dma_len(sg);
3780                dma_addr = sg_dma_address(sg);
3781
3782                if (len == 0)
3783                        break;
3784
3785                while ((count + 1) < sgt->nents) {
3786                        sg_next_iter = sg_next(sg);
3787                        len_next = sg_dma_len(sg_next_iter);
3788                        dma_addr_next = sg_dma_address(sg_next_iter);
3789
3790                        if (len_next == 0)
3791                                break;
3792
3793                        if ((dma_addr + len == dma_addr_next) &&
3794                                (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
3795                                len += len_next;
3796                                count++;
3797                                sg = sg_next_iter;
3798                        } else {
3799                                break;
3800                        }
3801                }
3802
3803                ctl = le32_to_cpu(user_dma_pkt->ctl);
3804                if (likely(dma_desc_cnt))
3805                        ctl &= ~GOYA_PKT_CTL_EB_MASK;
3806                ctl &= ~(GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK |
3807                                GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK);
3808                new_dma_pkt->ctl = cpu_to_le32(ctl);
3809                new_dma_pkt->tsize = cpu_to_le32((u32) len);
3810
3811                if (dir == DMA_TO_DEVICE) {
3812                        new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
3813                        new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
3814                } else {
3815                        new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);
3816                        new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);
3817                }
3818
3819                if (!user_memset)
3820                        device_memory_addr += len;
3821                dma_desc_cnt++;
3822                new_dma_pkt++;
3823        }
3824
3825        if (!dma_desc_cnt) {
3826                dev_err(hdev->dev,
3827                        "Error of 0 SG entries when patching DMA packet\n");
3828                return -EFAULT;
3829        }
3830
3831        /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */
3832        new_dma_pkt--;
3833        new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask);
3834
3835        *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);
3836
3837        return 0;
3838}
3839
3840static int goya_patch_cb(struct hl_device *hdev,
3841                                struct hl_cs_parser *parser)
3842{
3843        u32 cb_parsed_length = 0;
3844        u32 cb_patched_cur_length = 0;
3845        int rc = 0;
3846
3847        /* cb_user_size is more than 0 so loop will always be executed */
3848        while (cb_parsed_length < parser->user_cb_size) {
3849                enum packet_id pkt_id;
3850                u16 pkt_size;
3851                u32 new_pkt_size = 0;
3852                struct goya_packet *user_pkt, *kernel_pkt;
3853
3854                user_pkt = parser->user_cb->kernel_address + cb_parsed_length;
3855                kernel_pkt = parser->patched_cb->kernel_address +
3856                                        cb_patched_cur_length;
3857
3858                pkt_id = (enum packet_id) (
3859                                (le64_to_cpu(user_pkt->header) &
3860                                PACKET_HEADER_PACKET_ID_MASK) >>
3861                                        PACKET_HEADER_PACKET_ID_SHIFT);
3862
3863                if (!validate_packet_id(pkt_id)) {
3864                        dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
3865                        rc = -EINVAL;
3866                        break;
3867                }
3868
3869                pkt_size = goya_packet_sizes[pkt_id];
3870                cb_parsed_length += pkt_size;
3871                if (cb_parsed_length > parser->user_cb_size) {
3872                        dev_err(hdev->dev,
3873                                "packet 0x%x is out of CB boundary\n", pkt_id);
3874                        rc = -EINVAL;
3875                        break;
3876                }
3877
3878                switch (pkt_id) {
3879                case PACKET_LIN_DMA:
3880                        rc = goya_patch_dma_packet(hdev, parser,
3881                                        (struct packet_lin_dma *) user_pkt,
3882                                        (struct packet_lin_dma *) kernel_pkt,
3883                                        &new_pkt_size);
3884                        cb_patched_cur_length += new_pkt_size;
3885                        break;
3886
3887                case PACKET_WREG_32:
3888                        memcpy(kernel_pkt, user_pkt, pkt_size);
3889                        cb_patched_cur_length += pkt_size;
3890                        rc = goya_validate_wreg32(hdev, parser,
3891                                        (struct packet_wreg32 *) kernel_pkt);
3892                        break;
3893
3894                case PACKET_WREG_BULK:
3895                        dev_err(hdev->dev,
3896                                "User not allowed to use WREG_BULK\n");
3897                        rc = -EPERM;
3898                        break;
3899
3900                case PACKET_MSG_PROT:
3901                        dev_err(hdev->dev,
3902                                "User not allowed to use MSG_PROT\n");
3903                        rc = -EPERM;
3904                        break;
3905
3906                case PACKET_CP_DMA:
3907                        dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3908                        rc = -EPERM;
3909                        break;
3910
3911                case PACKET_STOP:
3912                        dev_err(hdev->dev, "User not allowed to use STOP\n");
3913                        rc = -EPERM;
3914                        break;
3915
3916                case PACKET_MSG_LONG:
3917                case PACKET_MSG_SHORT:
3918                case PACKET_FENCE:
3919                case PACKET_NOP:
3920                        memcpy(kernel_pkt, user_pkt, pkt_size);
3921                        cb_patched_cur_length += pkt_size;
3922                        break;
3923
3924                default:
3925                        dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3926                                pkt_id);
3927                        rc = -EINVAL;
3928                        break;
3929                }
3930
3931                if (rc)
3932                        break;
3933        }
3934
3935        return rc;
3936}
3937
3938static int goya_parse_cb_mmu(struct hl_device *hdev,
3939                struct hl_cs_parser *parser)
3940{
3941        u64 patched_cb_handle;
3942        u32 patched_cb_size;
3943        struct hl_cb *user_cb;
3944        int rc;
3945
3946        /*
3947         * The new CB should have space at the end for two MSG_PROT pkt:
3948         * 1. A packet that will act as a completion packet
3949         * 2. A packet that will generate MSI-X interrupt
3950         */
3951        parser->patched_cb_size = parser->user_cb_size +
3952                        sizeof(struct packet_msg_prot) * 2;
3953
3954        rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
3955                                parser->patched_cb_size, false, false,
3956                                &patched_cb_handle);
3957
3958        if (rc) {
3959                dev_err(hdev->dev,
3960                        "Failed to allocate patched CB for DMA CS %d\n",
3961                        rc);
3962                return rc;
3963        }
3964
3965        patched_cb_handle >>= PAGE_SHIFT;
3966        parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3967                                (u32) patched_cb_handle);
3968        /* hl_cb_get should never fail here */
3969        if (!parser->patched_cb) {
3970                dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n",
3971                        (u32) patched_cb_handle);
3972                rc = -EFAULT;
3973                goto out;
3974        }
3975
3976        /*
3977         * The check that parser->user_cb_size <= parser->user_cb->size was done
3978         * in validate_queue_index().
3979         */
3980        memcpy(parser->patched_cb->kernel_address,
3981                parser->user_cb->kernel_address,
3982                parser->user_cb_size);
3983
3984        patched_cb_size = parser->patched_cb_size;
3985
3986        /* validate patched CB instead of user CB */
3987        user_cb = parser->user_cb;
3988        parser->user_cb = parser->patched_cb;
3989        rc = goya_validate_cb(hdev, parser, true);
3990        parser->user_cb = user_cb;
3991
3992        if (rc) {
3993                hl_cb_put(parser->patched_cb);
3994                goto out;
3995        }
3996
3997        if (patched_cb_size != parser->patched_cb_size) {
3998                dev_err(hdev->dev, "user CB size mismatch\n");
3999                hl_cb_put(parser->patched_cb);
4000                rc = -EINVAL;
4001                goto out;
4002        }
4003
4004out:
4005        /*
4006         * Always call cb destroy here because we still have 1 reference
4007         * to it by calling cb_get earlier. After the job will be completed,
4008         * cb_put will release it, but here we want to remove it from the
4009         * idr
4010         */
4011        hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
4012                                        patched_cb_handle << PAGE_SHIFT);
4013
4014        return rc;
4015}
4016
4017static int goya_parse_cb_no_mmu(struct hl_device *hdev,
4018                                struct hl_cs_parser *parser)
4019{
4020        u64 patched_cb_handle;
4021        int rc;
4022
4023        rc = goya_validate_cb(hdev, parser, false);
4024
4025        if (rc)
4026                goto free_userptr;
4027
4028        rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
4029                                parser->patched_cb_size, false, false,
4030                                &patched_cb_handle);
4031        if (rc) {
4032                dev_err(hdev->dev,
4033                        "Failed to allocate patched CB for DMA CS %d\n", rc);
4034                goto free_userptr;
4035        }
4036
4037        patched_cb_handle >>= PAGE_SHIFT;
4038        parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
4039                                (u32) patched_cb_handle);
4040        /* hl_cb_get should never fail here */
4041        if (!parser->patched_cb) {
4042                dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n",
4043                        (u32) patched_cb_handle);
4044                rc = -EFAULT;
4045                goto out;
4046        }
4047
4048        rc = goya_patch_cb(hdev, parser);
4049
4050        if (rc)
4051                hl_cb_put(parser->patched_cb);
4052
4053out:
4054        /*
4055         * Always call cb destroy here because we still have 1 reference
4056         * to it by calling cb_get earlier. After the job will be completed,
4057         * cb_put will release it, but here we want to remove it from the
4058         * idr
4059         */
4060        hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
4061                                patched_cb_handle << PAGE_SHIFT);
4062
4063free_userptr:
4064        if (rc)
4065                hl_userptr_delete_list(hdev, parser->job_userptr_list);
4066        return rc;
4067}
4068
4069static int goya_parse_cb_no_ext_queue(struct hl_device *hdev,
4070                                        struct hl_cs_parser *parser)
4071{
4072        struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
4073        struct goya_device *goya = hdev->asic_specific;
4074
4075        if (goya->hw_cap_initialized & HW_CAP_MMU)
4076                return 0;
4077
4078        /* For internal queue jobs, just check if CB address is valid */
4079        if (hl_mem_area_inside_range(
4080                        (u64) (uintptr_t) parser->user_cb,
4081                        parser->user_cb_size,
4082                        asic_prop->sram_user_base_address,
4083                        asic_prop->sram_end_address))
4084                return 0;
4085
4086        if (hl_mem_area_inside_range(
4087                        (u64) (uintptr_t) parser->user_cb,
4088                        parser->user_cb_size,
4089                        asic_prop->dram_user_base_address,
4090                        asic_prop->dram_end_address))
4091                return 0;
4092
4093        dev_err(hdev->dev,
4094                "Internal CB address 0x%px + 0x%x is not in SRAM nor in DRAM\n",
4095                parser->user_cb, parser->user_cb_size);
4096
4097        return -EFAULT;
4098}
4099
4100int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
4101{
4102        struct goya_device *goya = hdev->asic_specific;
4103
4104        if (parser->queue_type == QUEUE_TYPE_INT)
4105                return goya_parse_cb_no_ext_queue(hdev, parser);
4106
4107        if (goya->hw_cap_initialized & HW_CAP_MMU)
4108                return goya_parse_cb_mmu(hdev, parser);
4109        else
4110                return goya_parse_cb_no_mmu(hdev, parser);
4111}
4112
4113void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address,
4114                                u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec,
4115                                bool eb)
4116{
4117        struct packet_msg_prot *cq_pkt;
4118        u32 tmp;
4119
4120        cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2);
4121
4122        tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
4123                        (1 << GOYA_PKT_CTL_EB_SHIFT) |
4124                        (1 << GOYA_PKT_CTL_MB_SHIFT);
4125        cq_pkt->ctl = cpu_to_le32(tmp);
4126        cq_pkt->value = cpu_to_le32(cq_val);
4127        cq_pkt->addr = cpu_to_le64(cq_addr);
4128
4129        cq_pkt++;
4130
4131        tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
4132                        (1 << GOYA_PKT_CTL_MB_SHIFT);
4133        cq_pkt->ctl = cpu_to_le32(tmp);
4134        cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF);
4135        cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF);
4136}
4137
4138void goya_update_eq_ci(struct hl_device *hdev, u32 val)
4139{
4140        WREG32(mmCPU_EQ_CI, val);
4141}
4142
4143void goya_restore_phase_topology(struct hl_device *hdev)
4144{
4145
4146}
4147
4148static void goya_clear_sm_regs(struct hl_device *hdev)
4149{
4150        int i, num_of_sob_in_longs, num_of_mon_in_longs;
4151
4152        num_of_sob_in_longs =
4153                ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4);
4154
4155        num_of_mon_in_longs =
4156                ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4);
4157
4158        for (i = 0 ; i < num_of_sob_in_longs ; i += 4)
4159                WREG32(mmSYNC_MNGR_SOB_OBJ_0 + i, 0);
4160
4161        for (i = 0 ; i < num_of_mon_in_longs ; i += 4)
4162                WREG32(mmSYNC_MNGR_MON_STATUS_0 + i, 0);
4163
4164        /* Flush all WREG to prevent race */
4165        i = RREG32(mmSYNC_MNGR_SOB_OBJ_0);
4166}
4167
4168/*
4169 * goya_debugfs_read32 - read a 32bit value from a given device or a host mapped
4170 *                       address.
4171 *
4172 * @hdev:       pointer to hl_device structure
4173 * @addr:       device or host mapped address
4174 * @val:        returned value
4175 *
4176 * In case of DDR address that is not mapped into the default aperture that
4177 * the DDR bar exposes, the function will configure the iATU so that the DDR
4178 * bar will be positioned at a base address that allows reading from the
4179 * required address. Configuring the iATU during normal operation can
4180 * lead to undefined behavior and therefore, should be done with extreme care
4181 *
4182 */
4183static int goya_debugfs_read32(struct hl_device *hdev, u64 addr,
4184                        bool user_address, u32 *val)
4185{
4186        struct asic_fixed_properties *prop = &hdev->asic_prop;
4187        u64 ddr_bar_addr, host_phys_end;