LXR linux/drivers/gpu/drm/radeon/cik.c

   1/*
   2 * Copyright 2012 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28#include <linux/slab.h>
  29
  30#include <drm/drm_vblank.h>
  31
  32#include "atom.h"
  33#include "evergreen.h"
  34#include "cik_blit_shaders.h"
  35#include "cik.h"
  36#include "cikd.h"
  37#include "clearstate_ci.h"
  38#include "r600.h"
  39#include "radeon.h"
  40#include "radeon_asic.h"
  41#include "radeon_audio.h"
  42#include "radeon_ucode.h"
  43#include "si.h"
  44#include "vce.h"
  45
  46#define SH_MEM_CONFIG_GFX_DEFAULT \
  47        ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
  48
  49MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  50MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  51MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  52MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  53MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  54MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  55MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  56MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  57MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  58
  59MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
  60MODULE_FIRMWARE("radeon/bonaire_me.bin");
  61MODULE_FIRMWARE("radeon/bonaire_ce.bin");
  62MODULE_FIRMWARE("radeon/bonaire_mec.bin");
  63MODULE_FIRMWARE("radeon/bonaire_mc.bin");
  64MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
  65MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
  66MODULE_FIRMWARE("radeon/bonaire_smc.bin");
  67MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
  68
  69MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  70MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  71MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  72MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  73MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  74MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  75MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  76MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  77MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  78
  79MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
  80MODULE_FIRMWARE("radeon/hawaii_me.bin");
  81MODULE_FIRMWARE("radeon/hawaii_ce.bin");
  82MODULE_FIRMWARE("radeon/hawaii_mec.bin");
  83MODULE_FIRMWARE("radeon/hawaii_mc.bin");
  84MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
  85MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
  86MODULE_FIRMWARE("radeon/hawaii_smc.bin");
  87MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
  88
  89MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  90MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  91MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  92MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  93MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  94MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  95
  96MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
  97MODULE_FIRMWARE("radeon/kaveri_me.bin");
  98MODULE_FIRMWARE("radeon/kaveri_ce.bin");
  99MODULE_FIRMWARE("radeon/kaveri_mec.bin");
 100MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
 101MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
 102MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
 103
 104MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
 105MODULE_FIRMWARE("radeon/KABINI_me.bin");
 106MODULE_FIRMWARE("radeon/KABINI_ce.bin");
 107MODULE_FIRMWARE("radeon/KABINI_mec.bin");
 108MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
 109MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
 110
 111MODULE_FIRMWARE("radeon/kabini_pfp.bin");
 112MODULE_FIRMWARE("radeon/kabini_me.bin");
 113MODULE_FIRMWARE("radeon/kabini_ce.bin");
 114MODULE_FIRMWARE("radeon/kabini_mec.bin");
 115MODULE_FIRMWARE("radeon/kabini_rlc.bin");
 116MODULE_FIRMWARE("radeon/kabini_sdma.bin");
 117
 118MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
 119MODULE_FIRMWARE("radeon/MULLINS_me.bin");
 120MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
 121MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
 122MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
 123MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
 124
 125MODULE_FIRMWARE("radeon/mullins_pfp.bin");
 126MODULE_FIRMWARE("radeon/mullins_me.bin");
 127MODULE_FIRMWARE("radeon/mullins_ce.bin");
 128MODULE_FIRMWARE("radeon/mullins_mec.bin");
 129MODULE_FIRMWARE("radeon/mullins_rlc.bin");
 130MODULE_FIRMWARE("radeon/mullins_sdma.bin");
 131
 132static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 133static void cik_rlc_stop(struct radeon_device *rdev);
 134static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 135static void cik_program_aspm(struct radeon_device *rdev);
 136static void cik_init_pg(struct radeon_device *rdev);
 137static void cik_init_cg(struct radeon_device *rdev);
 138static void cik_fini_pg(struct radeon_device *rdev);
 139static void cik_fini_cg(struct radeon_device *rdev);
 140static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
 141                                          bool enable);
 142
 143/**
 144 * cik_get_allowed_info_register - fetch the register for the info ioctl
 145 *
 146 * @rdev: radeon_device pointer
 147 * @reg: register offset in bytes
 148 * @val: register value
 149 *
 150 * Returns 0 for success or -EINVAL for an invalid register
 151 *
 152 */
 153int cik_get_allowed_info_register(struct radeon_device *rdev,
 154                                  u32 reg, u32 *val)
 155{
 156        switch (reg) {
 157        case GRBM_STATUS:
 158        case GRBM_STATUS2:
 159        case GRBM_STATUS_SE0:
 160        case GRBM_STATUS_SE1:
 161        case GRBM_STATUS_SE2:
 162        case GRBM_STATUS_SE3:
 163        case SRBM_STATUS:
 164        case SRBM_STATUS2:
 165        case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
 166        case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
 167        case UVD_STATUS:
 168        /* TODO VCE */
 169                *val = RREG32(reg);
 170                return 0;
 171        default:
 172                return -EINVAL;
 173        }
 174}
 175
 176/*
 177 * Indirect registers accessor
 178 */
 179u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
 180{
 181        unsigned long flags;
 182        u32 r;
 183
 184        spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 185        WREG32(CIK_DIDT_IND_INDEX, (reg));
 186        r = RREG32(CIK_DIDT_IND_DATA);
 187        spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 188        return r;
 189}
 190
 191void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 192{
 193        unsigned long flags;
 194
 195        spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 196        WREG32(CIK_DIDT_IND_INDEX, (reg));
 197        WREG32(CIK_DIDT_IND_DATA, (v));
 198        spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 199}
 200
 201/* get temperature in millidegrees */
 202int ci_get_temp(struct radeon_device *rdev)
 203{
 204        u32 temp;
 205        int actual_temp = 0;
 206
 207        temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 208                CTF_TEMP_SHIFT;
 209
 210        if (temp & 0x200)
 211                actual_temp = 255;
 212        else
 213                actual_temp = temp & 0x1ff;
 214
 215        return actual_temp * 1000;
 216}
 217
 218/* get temperature in millidegrees */
 219int kv_get_temp(struct radeon_device *rdev)
 220{
 221        u32 temp;
 222        int actual_temp = 0;
 223
 224        temp = RREG32_SMC(0xC0300E0C);
 225
 226        if (temp)
 227                actual_temp = (temp / 8) - 49;
 228        else
 229                actual_temp = 0;
 230
 231        return actual_temp * 1000;
 232}
 233
 234/*
 235 * Indirect registers accessor
 236 */
 237u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 238{
 239        unsigned long flags;
 240        u32 r;
 241
 242        spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 243        WREG32(PCIE_INDEX, reg);
 244        (void)RREG32(PCIE_INDEX);
 245        r = RREG32(PCIE_DATA);
 246        spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 247        return r;
 248}
 249
 250void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 251{
 252        unsigned long flags;
 253
 254        spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 255        WREG32(PCIE_INDEX, reg);
 256        (void)RREG32(PCIE_INDEX);
 257        WREG32(PCIE_DATA, v);
 258        (void)RREG32(PCIE_DATA);
 259        spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 260}
 261
 262static const u32 spectre_rlc_save_restore_register_list[] =
 263{
 264        (0x0e00 << 16) | (0xc12c >> 2),
 265        0x00000000,
 266        (0x0e00 << 16) | (0xc140 >> 2),
 267        0x00000000,
 268        (0x0e00 << 16) | (0xc150 >> 2),
 269        0x00000000,
 270        (0x0e00 << 16) | (0xc15c >> 2),
 271        0x00000000,
 272        (0x0e00 << 16) | (0xc168 >> 2),
 273        0x00000000,
 274        (0x0e00 << 16) | (0xc170 >> 2),
 275        0x00000000,
 276        (0x0e00 << 16) | (0xc178 >> 2),
 277        0x00000000,
 278        (0x0e00 << 16) | (0xc204 >> 2),
 279        0x00000000,
 280        (0x0e00 << 16) | (0xc2b4 >> 2),
 281        0x00000000,
 282        (0x0e00 << 16) | (0xc2b8 >> 2),
 283        0x00000000,
 284        (0x0e00 << 16) | (0xc2bc >> 2),
 285        0x00000000,
 286        (0x0e00 << 16) | (0xc2c0 >> 2),
 287        0x00000000,
 288        (0x0e00 << 16) | (0x8228 >> 2),
 289        0x00000000,
 290        (0x0e00 << 16) | (0x829c >> 2),
 291        0x00000000,
 292        (0x0e00 << 16) | (0x869c >> 2),
 293        0x00000000,
 294        (0x0600 << 16) | (0x98f4 >> 2),
 295        0x00000000,
 296        (0x0e00 << 16) | (0x98f8 >> 2),
 297        0x00000000,
 298        (0x0e00 << 16) | (0x9900 >> 2),
 299        0x00000000,
 300        (0x0e00 << 16) | (0xc260 >> 2),
 301        0x00000000,
 302        (0x0e00 << 16) | (0x90e8 >> 2),
 303        0x00000000,
 304        (0x0e00 << 16) | (0x3c000 >> 2),
 305        0x00000000,
 306        (0x0e00 << 16) | (0x3c00c >> 2),
 307        0x00000000,
 308        (0x0e00 << 16) | (0x8c1c >> 2),
 309        0x00000000,
 310        (0x0e00 << 16) | (0x9700 >> 2),
 311        0x00000000,
 312        (0x0e00 << 16) | (0xcd20 >> 2),
 313        0x00000000,
 314        (0x4e00 << 16) | (0xcd20 >> 2),
 315        0x00000000,
 316        (0x5e00 << 16) | (0xcd20 >> 2),
 317        0x00000000,
 318        (0x6e00 << 16) | (0xcd20 >> 2),
 319        0x00000000,
 320        (0x7e00 << 16) | (0xcd20 >> 2),
 321        0x00000000,
 322        (0x8e00 << 16) | (0xcd20 >> 2),
 323        0x00000000,
 324        (0x9e00 << 16) | (0xcd20 >> 2),
 325        0x00000000,
 326        (0xae00 << 16) | (0xcd20 >> 2),
 327        0x00000000,
 328        (0xbe00 << 16) | (0xcd20 >> 2),
 329        0x00000000,
 330        (0x0e00 << 16) | (0x89bc >> 2),
 331        0x00000000,
 332        (0x0e00 << 16) | (0x8900 >> 2),
 333        0x00000000,
 334        0x3,
 335        (0x0e00 << 16) | (0xc130 >> 2),
 336        0x00000000,
 337        (0x0e00 << 16) | (0xc134 >> 2),
 338        0x00000000,
 339        (0x0e00 << 16) | (0xc1fc >> 2),
 340        0x00000000,
 341        (0x0e00 << 16) | (0xc208 >> 2),
 342        0x00000000,
 343        (0x0e00 << 16) | (0xc264 >> 2),
 344        0x00000000,
 345        (0x0e00 << 16) | (0xc268 >> 2),
 346        0x00000000,
 347        (0x0e00 << 16) | (0xc26c >> 2),
 348        0x00000000,
 349        (0x0e00 << 16) | (0xc270 >> 2),
 350        0x00000000,
 351        (0x0e00 << 16) | (0xc274 >> 2),
 352        0x00000000,
 353        (0x0e00 << 16) | (0xc278 >> 2),
 354        0x00000000,
 355        (0x0e00 << 16) | (0xc27c >> 2),
 356        0x00000000,
 357        (0x0e00 << 16) | (0xc280 >> 2),
 358        0x00000000,
 359        (0x0e00 << 16) | (0xc284 >> 2),
 360        0x00000000,
 361        (0x0e00 << 16) | (0xc288 >> 2),
 362        0x00000000,
 363        (0x0e00 << 16) | (0xc28c >> 2),
 364        0x00000000,
 365        (0x0e00 << 16) | (0xc290 >> 2),
 366        0x00000000,
 367        (0x0e00 << 16) | (0xc294 >> 2),
 368        0x00000000,
 369        (0x0e00 << 16) | (0xc298 >> 2),
 370        0x00000000,
 371        (0x0e00 << 16) | (0xc29c >> 2),
 372        0x00000000,
 373        (0x0e00 << 16) | (0xc2a0 >> 2),
 374        0x00000000,
 375        (0x0e00 << 16) | (0xc2a4 >> 2),
 376        0x00000000,
 377        (0x0e00 << 16) | (0xc2a8 >> 2),
 378        0x00000000,
 379        (0x0e00 << 16) | (0xc2ac  >> 2),
 380        0x00000000,
 381        (0x0e00 << 16) | (0xc2b0 >> 2),
 382        0x00000000,
 383        (0x0e00 << 16) | (0x301d0 >> 2),
 384        0x00000000,
 385        (0x0e00 << 16) | (0x30238 >> 2),
 386        0x00000000,
 387        (0x0e00 << 16) | (0x30250 >> 2),
 388        0x00000000,
 389        (0x0e00 << 16) | (0x30254 >> 2),
 390        0x00000000,
 391        (0x0e00 << 16) | (0x30258 >> 2),
 392        0x00000000,
 393        (0x0e00 << 16) | (0x3025c >> 2),
 394        0x00000000,
 395        (0x4e00 << 16) | (0xc900 >> 2),
 396        0x00000000,
 397        (0x5e00 << 16) | (0xc900 >> 2),
 398        0x00000000,
 399        (0x6e00 << 16) | (0xc900 >> 2),
 400        0x00000000,
 401        (0x7e00 << 16) | (0xc900 >> 2),
 402        0x00000000,
 403        (0x8e00 << 16) | (0xc900 >> 2),
 404        0x00000000,
 405        (0x9e00 << 16) | (0xc900 >> 2),
 406        0x00000000,
 407        (0xae00 << 16) | (0xc900 >> 2),
 408        0x00000000,
 409        (0xbe00 << 16) | (0xc900 >> 2),
 410        0x00000000,
 411        (0x4e00 << 16) | (0xc904 >> 2),
 412        0x00000000,
 413        (0x5e00 << 16) | (0xc904 >> 2),
 414        0x00000000,
 415        (0x6e00 << 16) | (0xc904 >> 2),
 416        0x00000000,
 417        (0x7e00 << 16) | (0xc904 >> 2),
 418        0x00000000,
 419        (0x8e00 << 16) | (0xc904 >> 2),
 420        0x00000000,
 421        (0x9e00 << 16) | (0xc904 >> 2),
 422        0x00000000,
 423        (0xae00 << 16) | (0xc904 >> 2),
 424        0x00000000,
 425        (0xbe00 << 16) | (0xc904 >> 2),
 426        0x00000000,
 427        (0x4e00 << 16) | (0xc908 >> 2),
 428        0x00000000,
 429        (0x5e00 << 16) | (0xc908 >> 2),
 430        0x00000000,
 431        (0x6e00 << 16) | (0xc908 >> 2),
 432        0x00000000,
 433        (0x7e00 << 16) | (0xc908 >> 2),
 434        0x00000000,
 435        (0x8e00 << 16) | (0xc908 >> 2),
 436        0x00000000,
 437        (0x9e00 << 16) | (0xc908 >> 2),
 438        0x00000000,
 439        (0xae00 << 16) | (0xc908 >> 2),
 440        0x00000000,
 441        (0xbe00 << 16) | (0xc908 >> 2),
 442        0x00000000,
 443        (0x4e00 << 16) | (0xc90c >> 2),
 444        0x00000000,
 445        (0x5e00 << 16) | (0xc90c >> 2),
 446        0x00000000,
 447        (0x6e00 << 16) | (0xc90c >> 2),
 448        0x00000000,
 449        (0x7e00 << 16) | (0xc90c >> 2),
 450        0x00000000,
 451        (0x8e00 << 16) | (0xc90c >> 2),
 452        0x00000000,
 453        (0x9e00 << 16) | (0xc90c >> 2),
 454        0x00000000,
 455        (0xae00 << 16) | (0xc90c >> 2),
 456        0x00000000,
 457        (0xbe00 << 16) | (0xc90c >> 2),
 458        0x00000000,
 459        (0x4e00 << 16) | (0xc910 >> 2),
 460        0x00000000,
 461        (0x5e00 << 16) | (0xc910 >> 2),
 462        0x00000000,
 463        (0x6e00 << 16) | (0xc910 >> 2),
 464        0x00000000,
 465        (0x7e00 << 16) | (0xc910 >> 2),
 466        0x00000000,
 467        (0x8e00 << 16) | (0xc910 >> 2),
 468        0x00000000,
 469        (0x9e00 << 16) | (0xc910 >> 2),
 470        0x00000000,
 471        (0xae00 << 16) | (0xc910 >> 2),
 472        0x00000000,
 473        (0xbe00 << 16) | (0xc910 >> 2),
 474        0x00000000,
 475        (0x0e00 << 16) | (0xc99c >> 2),
 476        0x00000000,
 477        (0x0e00 << 16) | (0x9834 >> 2),
 478        0x00000000,
 479        (0x0000 << 16) | (0x30f00 >> 2),
 480        0x00000000,
 481        (0x0001 << 16) | (0x30f00 >> 2),
 482        0x00000000,
 483        (0x0000 << 16) | (0x30f04 >> 2),
 484        0x00000000,
 485        (0x0001 << 16) | (0x30f04 >> 2),
 486        0x00000000,
 487        (0x0000 << 16) | (0x30f08 >> 2),
 488        0x00000000,
 489        (0x0001 << 16) | (0x30f08 >> 2),
 490        0x00000000,
 491        (0x0000 << 16) | (0x30f0c >> 2),
 492        0x00000000,
 493        (0x0001 << 16) | (0x30f0c >> 2),
 494        0x00000000,
 495        (0x0600 << 16) | (0x9b7c >> 2),
 496        0x00000000,
 497        (0x0e00 << 16) | (0x8a14 >> 2),
 498        0x00000000,
 499        (0x0e00 << 16) | (0x8a18 >> 2),
 500        0x00000000,
 501        (0x0600 << 16) | (0x30a00 >> 2),
 502        0x00000000,
 503        (0x0e00 << 16) | (0x8bf0 >> 2),
 504        0x00000000,
 505        (0x0e00 << 16) | (0x8bcc >> 2),
 506        0x00000000,
 507        (0x0e00 << 16) | (0x8b24 >> 2),
 508        0x00000000,
 509        (0x0e00 << 16) | (0x30a04 >> 2),
 510        0x00000000,
 511        (0x0600 << 16) | (0x30a10 >> 2),
 512        0x00000000,
 513        (0x0600 << 16) | (0x30a14 >> 2),
 514        0x00000000,
 515        (0x0600 << 16) | (0x30a18 >> 2),
 516        0x00000000,
 517        (0x0600 << 16) | (0x30a2c >> 2),
 518        0x00000000,
 519        (0x0e00 << 16) | (0xc700 >> 2),
 520        0x00000000,
 521        (0x0e00 << 16) | (0xc704 >> 2),
 522        0x00000000,
 523        (0x0e00 << 16) | (0xc708 >> 2),
 524        0x00000000,
 525        (0x0e00 << 16) | (0xc768 >> 2),
 526        0x00000000,
 527        (0x0400 << 16) | (0xc770 >> 2),
 528        0x00000000,
 529        (0x0400 << 16) | (0xc774 >> 2),
 530        0x00000000,
 531        (0x0400 << 16) | (0xc778 >> 2),
 532        0x00000000,
 533        (0x0400 << 16) | (0xc77c >> 2),
 534        0x00000000,
 535        (0x0400 << 16) | (0xc780 >> 2),
 536        0x00000000,
 537        (0x0400 << 16) | (0xc784 >> 2),
 538        0x00000000,
 539        (0x0400 << 16) | (0xc788 >> 2),
 540        0x00000000,
 541        (0x0400 << 16) | (0xc78c >> 2),
 542        0x00000000,
 543        (0x0400 << 16) | (0xc798 >> 2),
 544        0x00000000,
 545        (0x0400 << 16) | (0xc79c >> 2),
 546        0x00000000,
 547        (0x0400 << 16) | (0xc7a0 >> 2),
 548        0x00000000,
 549        (0x0400 << 16) | (0xc7a4 >> 2),
 550        0x00000000,
 551        (0x0400 << 16) | (0xc7a8 >> 2),
 552        0x00000000,
 553        (0x0400 << 16) | (0xc7ac >> 2),
 554        0x00000000,
 555        (0x0400 << 16) | (0xc7b0 >> 2),
 556        0x00000000,
 557        (0x0400 << 16) | (0xc7b4 >> 2),
 558        0x00000000,
 559        (0x0e00 << 16) | (0x9100 >> 2),
 560        0x00000000,
 561        (0x0e00 << 16) | (0x3c010 >> 2),
 562        0x00000000,
 563        (0x0e00 << 16) | (0x92a8 >> 2),
 564        0x00000000,
 565        (0x0e00 << 16) | (0x92ac >> 2),
 566        0x00000000,
 567        (0x0e00 << 16) | (0x92b4 >> 2),
 568        0x00000000,
 569        (0x0e00 << 16) | (0x92b8 >> 2),
 570        0x00000000,
 571        (0x0e00 << 16) | (0x92bc >> 2),
 572        0x00000000,
 573        (0x0e00 << 16) | (0x92c0 >> 2),
 574        0x00000000,
 575        (0x0e00 << 16) | (0x92c4 >> 2),
 576        0x00000000,
 577        (0x0e00 << 16) | (0x92c8 >> 2),
 578        0x00000000,
 579        (0x0e00 << 16) | (0x92cc >> 2),
 580        0x00000000,
 581        (0x0e00 << 16) | (0x92d0 >> 2),
 582        0x00000000,
 583        (0x0e00 << 16) | (0x8c00 >> 2),
 584        0x00000000,
 585        (0x0e00 << 16) | (0x8c04 >> 2),
 586        0x00000000,
 587        (0x0e00 << 16) | (0x8c20 >> 2),
 588        0x00000000,
 589        (0x0e00 << 16) | (0x8c38 >> 2),
 590        0x00000000,
 591        (0x0e00 << 16) | (0x8c3c >> 2),
 592        0x00000000,
 593        (0x0e00 << 16) | (0xae00 >> 2),
 594        0x00000000,
 595        (0x0e00 << 16) | (0x9604 >> 2),
 596        0x00000000,
 597        (0x0e00 << 16) | (0xac08 >> 2),
 598        0x00000000,
 599        (0x0e00 << 16) | (0xac0c >> 2),
 600        0x00000000,
 601        (0x0e00 << 16) | (0xac10 >> 2),
 602        0x00000000,
 603        (0x0e00 << 16) | (0xac14 >> 2),
 604        0x00000000,
 605        (0x0e00 << 16) | (0xac58 >> 2),
 606        0x00000000,
 607        (0x0e00 << 16) | (0xac68 >> 2),
 608        0x00000000,
 609        (0x0e00 << 16) | (0xac6c >> 2),
 610        0x00000000,
 611        (0x0e00 << 16) | (0xac70 >> 2),
 612        0x00000000,
 613        (0x0e00 << 16) | (0xac74 >> 2),
 614        0x00000000,
 615        (0x0e00 << 16) | (0xac78 >> 2),
 616        0x00000000,
 617        (0x0e00 << 16) | (0xac7c >> 2),
 618        0x00000000,
 619        (0x0e00 << 16) | (0xac80 >> 2),
 620        0x00000000,
 621        (0x0e00 << 16) | (0xac84 >> 2),
 622        0x00000000,
 623        (0x0e00 << 16) | (0xac88 >> 2),
 624        0x00000000,
 625        (0x0e00 << 16) | (0xac8c >> 2),
 626        0x00000000,
 627        (0x0e00 << 16) | (0x970c >> 2),
 628        0x00000000,
 629        (0x0e00 << 16) | (0x9714 >> 2),
 630        0x00000000,
 631        (0x0e00 << 16) | (0x9718 >> 2),
 632        0x00000000,
 633        (0x0e00 << 16) | (0x971c >> 2),
 634        0x00000000,
 635        (0x0e00 << 16) | (0x31068 >> 2),
 636        0x00000000,
 637        (0x4e00 << 16) | (0x31068 >> 2),
 638        0x00000000,
 639        (0x5e00 << 16) | (0x31068 >> 2),
 640        0x00000000,
 641        (0x6e00 << 16) | (0x31068 >> 2),
 642        0x00000000,
 643        (0x7e00 << 16) | (0x31068 >> 2),
 644        0x00000000,
 645        (0x8e00 << 16) | (0x31068 >> 2),
 646        0x00000000,
 647        (0x9e00 << 16) | (0x31068 >> 2),
 648        0x00000000,
 649        (0xae00 << 16) | (0x31068 >> 2),
 650        0x00000000,
 651        (0xbe00 << 16) | (0x31068 >> 2),
 652        0x00000000,
 653        (0x0e00 << 16) | (0xcd10 >> 2),
 654        0x00000000,
 655        (0x0e00 << 16) | (0xcd14 >> 2),
 656        0x00000000,
 657        (0x0e00 << 16) | (0x88b0 >> 2),
 658        0x00000000,
 659        (0x0e00 << 16) | (0x88b4 >> 2),
 660        0x00000000,
 661        (0x0e00 << 16) | (0x88b8 >> 2),
 662        0x00000000,
 663        (0x0e00 << 16) | (0x88bc >> 2),
 664        0x00000000,
 665        (0x0400 << 16) | (0x89c0 >> 2),
 666        0x00000000,
 667        (0x0e00 << 16) | (0x88c4 >> 2),
 668        0x00000000,
 669        (0x0e00 << 16) | (0x88c8 >> 2),
 670        0x00000000,
 671        (0x0e00 << 16) | (0x88d0 >> 2),
 672        0x00000000,
 673        (0x0e00 << 16) | (0x88d4 >> 2),
 674        0x00000000,
 675        (0x0e00 << 16) | (0x88d8 >> 2),
 676        0x00000000,
 677        (0x0e00 << 16) | (0x8980 >> 2),
 678        0x00000000,
 679        (0x0e00 << 16) | (0x30938 >> 2),
 680        0x00000000,
 681        (0x0e00 << 16) | (0x3093c >> 2),
 682        0x00000000,
 683        (0x0e00 << 16) | (0x30940 >> 2),
 684        0x00000000,
 685        (0x0e00 << 16) | (0x89a0 >> 2),
 686        0x00000000,
 687        (0x0e00 << 16) | (0x30900 >> 2),
 688        0x00000000,
 689        (0x0e00 << 16) | (0x30904 >> 2),
 690        0x00000000,
 691        (0x0e00 << 16) | (0x89b4 >> 2),
 692        0x00000000,
 693        (0x0e00 << 16) | (0x3c210 >> 2),
 694        0x00000000,
 695        (0x0e00 << 16) | (0x3c214 >> 2),
 696        0x00000000,
 697        (0x0e00 << 16) | (0x3c218 >> 2),
 698        0x00000000,
 699        (0x0e00 << 16) | (0x8904 >> 2),
 700        0x00000000,
 701        0x5,
 702        (0x0e00 << 16) | (0x8c28 >> 2),
 703        (0x0e00 << 16) | (0x8c2c >> 2),
 704        (0x0e00 << 16) | (0x8c30 >> 2),
 705        (0x0e00 << 16) | (0x8c34 >> 2),
 706        (0x0e00 << 16) | (0x9600 >> 2),
 707};
 708
 709static const u32 kalindi_rlc_save_restore_register_list[] =
 710{
 711        (0x0e00 << 16) | (0xc12c >> 2),
 712        0x00000000,
 713        (0x0e00 << 16) | (0xc140 >> 2),
 714        0x00000000,
 715        (0x0e00 << 16) | (0xc150 >> 2),
 716        0x00000000,
 717        (0x0e00 << 16) | (0xc15c >> 2),
 718        0x00000000,
 719        (0x0e00 << 16) | (0xc168 >> 2),
 720        0x00000000,
 721        (0x0e00 << 16) | (0xc170 >> 2),
 722        0x00000000,
 723        (0x0e00 << 16) | (0xc204 >> 2),
 724        0x00000000,
 725        (0x0e00 << 16) | (0xc2b4 >> 2),
 726        0x00000000,
 727        (0x0e00 << 16) | (0xc2b8 >> 2),
 728        0x00000000,
 729        (0x0e00 << 16) | (0xc2bc >> 2),
 730        0x00000000,
 731        (0x0e00 << 16) | (0xc2c0 >> 2),
 732        0x00000000,
 733        (0x0e00 << 16) | (0x8228 >> 2),
 734        0x00000000,
 735        (0x0e00 << 16) | (0x829c >> 2),
 736        0x00000000,
 737        (0x0e00 << 16) | (0x869c >> 2),
 738        0x00000000,
 739        (0x0600 << 16) | (0x98f4 >> 2),
 740        0x00000000,
 741        (0x0e00 << 16) | (0x98f8 >> 2),
 742        0x00000000,
 743        (0x0e00 << 16) | (0x9900 >> 2),
 744        0x00000000,
 745        (0x0e00 << 16) | (0xc260 >> 2),
 746        0x00000000,
 747        (0x0e00 << 16) | (0x90e8 >> 2),
 748        0x00000000,
 749        (0x0e00 << 16) | (0x3c000 >> 2),
 750        0x00000000,
 751        (0x0e00 << 16) | (0x3c00c >> 2),
 752        0x00000000,
 753        (0x0e00 << 16) | (0x8c1c >> 2),
 754        0x00000000,
 755        (0x0e00 << 16) | (0x9700 >> 2),
 756        0x00000000,
 757        (0x0e00 << 16) | (0xcd20 >> 2),
 758        0x00000000,
 759        (0x4e00 << 16) | (0xcd20 >> 2),
 760        0x00000000,
 761        (0x5e00 << 16) | (0xcd20 >> 2),
 762        0x00000000,
 763        (0x6e00 << 16) | (0xcd20 >> 2),
 764        0x00000000,
 765        (0x7e00 << 16) | (0xcd20 >> 2),
 766        0x00000000,
 767        (0x0e00 << 16) | (0x89bc >> 2),
 768        0x00000000,
 769        (0x0e00 << 16) | (0x8900 >> 2),
 770        0x00000000,
 771        0x3,
 772        (0x0e00 << 16) | (0xc130 >> 2),
 773        0x00000000,
 774        (0x0e00 << 16) | (0xc134 >> 2),
 775        0x00000000,
 776        (0x0e00 << 16) | (0xc1fc >> 2),
 777        0x00000000,
 778        (0x0e00 << 16) | (0xc208 >> 2),
 779        0x00000000,
 780        (0x0e00 << 16) | (0xc264 >> 2),
 781        0x00000000,
 782        (0x0e00 << 16) | (0xc268 >> 2),
 783        0x00000000,
 784        (0x0e00 << 16) | (0xc26c >> 2),
 785        0x00000000,
 786        (0x0e00 << 16) | (0xc270 >> 2),
 787        0x00000000,
 788        (0x0e00 << 16) | (0xc274 >> 2),
 789        0x00000000,
 790        (0x0e00 << 16) | (0xc28c >> 2),
 791        0x00000000,
 792        (0x0e00 << 16) | (0xc290 >> 2),
 793        0x00000000,
 794        (0x0e00 << 16) | (0xc294 >> 2),
 795        0x00000000,
 796        (0x0e00 << 16) | (0xc298 >> 2),
 797        0x00000000,
 798        (0x0e00 << 16) | (0xc2a0 >> 2),
 799        0x00000000,
 800        (0x0e00 << 16) | (0xc2a4 >> 2),
 801        0x00000000,
 802        (0x0e00 << 16) | (0xc2a8 >> 2),
 803        0x00000000,
 804        (0x0e00 << 16) | (0xc2ac >> 2),
 805        0x00000000,
 806        (0x0e00 << 16) | (0x301d0 >> 2),
 807        0x00000000,
 808        (0x0e00 << 16) | (0x30238 >> 2),
 809        0x00000000,
 810        (0x0e00 << 16) | (0x30250 >> 2),
 811        0x00000000,
 812        (0x0e00 << 16) | (0x30254 >> 2),
 813        0x00000000,
 814        (0x0e00 << 16) | (0x30258 >> 2),
 815        0x00000000,
 816        (0x0e00 << 16) | (0x3025c >> 2),
 817        0x00000000,
 818        (0x4e00 << 16) | (0xc900 >> 2),
 819        0x00000000,
 820        (0x5e00 << 16) | (0xc900 >> 2),
 821        0x00000000,
 822        (0x6e00 << 16) | (0xc900 >> 2),
 823        0x00000000,
 824        (0x7e00 << 16) | (0xc900 >> 2),
 825        0x00000000,
 826        (0x4e00 << 16) | (0xc904 >> 2),
 827        0x00000000,
 828        (0x5e00 << 16) | (0xc904 >> 2),
 829        0x00000000,
 830        (0x6e00 << 16) | (0xc904 >> 2),
 831        0x00000000,
 832        (0x7e00 << 16) | (0xc904 >> 2),
 833        0x00000000,
 834        (0x4e00 << 16) | (0xc908 >> 2),
 835        0x00000000,
 836        (0x5e00 << 16) | (0xc908 >> 2),
 837        0x00000000,
 838        (0x6e00 << 16) | (0xc908 >> 2),
 839        0x00000000,
 840        (0x7e00 << 16) | (0xc908 >> 2),
 841        0x00000000,
 842        (0x4e00 << 16) | (0xc90c >> 2),
 843        0x00000000,
 844        (0x5e00 << 16) | (0xc90c >> 2),
 845        0x00000000,
 846        (0x6e00 << 16) | (0xc90c >> 2),
 847        0x00000000,
 848        (0x7e00 << 16) | (0xc90c >> 2),
 849        0x00000000,
 850        (0x4e00 << 16) | (0xc910 >> 2),
 851        0x00000000,
 852        (0x5e00 << 16) | (0xc910 >> 2),
 853        0x00000000,
 854        (0x6e00 << 16) | (0xc910 >> 2),
 855        0x00000000,
 856        (0x7e00 << 16) | (0xc910 >> 2),
 857        0x00000000,
 858        (0x0e00 << 16) | (0xc99c >> 2),
 859        0x00000000,
 860        (0x0e00 << 16) | (0x9834 >> 2),
 861        0x00000000,
 862        (0x0000 << 16) | (0x30f00 >> 2),
 863        0x00000000,
 864        (0x0000 << 16) | (0x30f04 >> 2),
 865        0x00000000,
 866        (0x0000 << 16) | (0x30f08 >> 2),
 867        0x00000000,
 868        (0x0000 << 16) | (0x30f0c >> 2),
 869        0x00000000,
 870        (0x0600 << 16) | (0x9b7c >> 2),
 871        0x00000000,
 872        (0x0e00 << 16) | (0x8a14 >> 2),
 873        0x00000000,
 874        (0x0e00 << 16) | (0x8a18 >> 2),
 875        0x00000000,
 876        (0x0600 << 16) | (0x30a00 >> 2),
 877        0x00000000,
 878        (0x0e00 << 16) | (0x8bf0 >> 2),
 879        0x00000000,
 880        (0x0e00 << 16) | (0x8bcc >> 2),
 881        0x00000000,
 882        (0x0e00 << 16) | (0x8b24 >> 2),
 883        0x00000000,
 884        (0x0e00 << 16) | (0x30a04 >> 2),
 885        0x00000000,
 886        (0x0600 << 16) | (0x30a10 >> 2),
 887        0x00000000,
 888        (0x0600 << 16) | (0x30a14 >> 2),
 889        0x00000000,
 890        (0x0600 << 16) | (0x30a18 >> 2),
 891        0x00000000,
 892        (0x0600 << 16) | (0x30a2c >> 2),
 893        0x00000000,
 894        (0x0e00 << 16) | (0xc700 >> 2),
 895        0x00000000,
 896        (0x0e00 << 16) | (0xc704 >> 2),
 897        0x00000000,
 898        (0x0e00 << 16) | (0xc708 >> 2),
 899        0x00000000,
 900        (0x0e00 << 16) | (0xc768 >> 2),
 901        0x00000000,
 902        (0x0400 << 16) | (0xc770 >> 2),
 903        0x00000000,
 904        (0x0400 << 16) | (0xc774 >> 2),
 905        0x00000000,
 906        (0x0400 << 16) | (0xc798 >> 2),
 907        0x00000000,
 908        (0x0400 << 16) | (0xc79c >> 2),
 909        0x00000000,
 910        (0x0e00 << 16) | (0x9100 >> 2),
 911        0x00000000,
 912        (0x0e00 << 16) | (0x3c010 >> 2),
 913        0x00000000,
 914        (0x0e00 << 16) | (0x8c00 >> 2),
 915        0x00000000,
 916        (0x0e00 << 16) | (0x8c04 >> 2),
 917        0x00000000,
 918        (0x0e00 << 16) | (0x8c20 >> 2),
 919        0x00000000,
 920        (0x0e00 << 16) | (0x8c38 >> 2),
 921        0x00000000,
 922        (0x0e00 << 16) | (0x8c3c >> 2),
 923        0x00000000,
 924        (0x0e00 << 16) | (0xae00 >> 2),
 925        0x00000000,
 926        (0x0e00 << 16) | (0x9604 >> 2),
 927        0x00000000,
 928        (0x0e00 << 16) | (0xac08 >> 2),
 929        0x00000000,
 930        (0x0e00 << 16) | (0xac0c >> 2),
 931        0x00000000,
 932        (0x0e00 << 16) | (0xac10 >> 2),
 933        0x00000000,
 934        (0x0e00 << 16) | (0xac14 >> 2),
 935        0x00000000,
 936        (0x0e00 << 16) | (0xac58 >> 2),
 937        0x00000000,
 938        (0x0e00 << 16) | (0xac68 >> 2),
 939        0x00000000,
 940        (0x0e00 << 16) | (0xac6c >> 2),
 941        0x00000000,
 942        (0x0e00 << 16) | (0xac70 >> 2),
 943        0x00000000,
 944        (0x0e00 << 16) | (0xac74 >> 2),
 945        0x00000000,
 946        (0x0e00 << 16) | (0xac78 >> 2),
 947        0x00000000,
 948        (0x0e00 << 16) | (0xac7c >> 2),
 949        0x00000000,
 950        (0x0e00 << 16) | (0xac80 >> 2),
 951        0x00000000,
 952        (0x0e00 << 16) | (0xac84 >> 2),
 953        0x00000000,
 954        (0x0e00 << 16) | (0xac88 >> 2),
 955        0x00000000,
 956        (0x0e00 << 16) | (0xac8c >> 2),
 957        0x00000000,
 958        (0x0e00 << 16) | (0x970c >> 2),
 959        0x00000000,
 960        (0x0e00 << 16) | (0x9714 >> 2),
 961        0x00000000,
 962        (0x0e00 << 16) | (0x9718 >> 2),
 963        0x00000000,
 964        (0x0e00 << 16) | (0x971c >> 2),
 965        0x00000000,
 966        (0x0e00 << 16) | (0x31068 >> 2),
 967        0x00000000,
 968        (0x4e00 << 16) | (0x31068 >> 2),
 969        0x00000000,
 970        (0x5e00 << 16) | (0x31068 >> 2),
 971        0x00000000,
 972        (0x6e00 << 16) | (0x31068 >> 2),
 973        0x00000000,
 974        (0x7e00 << 16) | (0x31068 >> 2),
 975        0x00000000,
 976        (0x0e00 << 16) | (0xcd10 >> 2),
 977        0x00000000,
 978        (0x0e00 << 16) | (0xcd14 >> 2),
 979        0x00000000,
 980        (0x0e00 << 16) | (0x88b0 >> 2),
 981        0x00000000,
 982        (0x0e00 << 16) | (0x88b4 >> 2),
 983        0x00000000,
 984        (0x0e00 << 16) | (0x88b8 >> 2),
 985        0x00000000,
 986        (0x0e00 << 16) | (0x88bc >> 2),
 987        0x00000000,
 988        (0x0400 << 16) | (0x89c0 >> 2),
 989        0x00000000,
 990        (0x0e00 << 16) | (0x88c4 >> 2),
 991        0x00000000,
 992        (0x0e00 << 16) | (0x88c8 >> 2),
 993        0x00000000,
 994        (0x0e00 << 16) | (0x88d0 >> 2),
 995        0x00000000,
 996        (0x0e00 << 16) | (0x88d4 >> 2),
 997        0x00000000,
 998        (0x0e00 << 16) | (0x88d8 >> 2),
 999        0x00000000,
1000        (0x0e00 << 16) | (0x8980 >> 2),

1001        0x00000000,
1002        (0x0e00 << 16) | (0x30938 >> 2),
1003        0x00000000,
1004        (0x0e00 << 16) | (0x3093c >> 2),
1005        0x00000000,
1006        (0x0e00 << 16) | (0x30940 >> 2),
1007        0x00000000,
1008        (0x0e00 << 16) | (0x89a0 >> 2),
1009        0x00000000,
1010        (0x0e00 << 16) | (0x30900 >> 2),
1011        0x00000000,
1012        (0x0e00 << 16) | (0x30904 >> 2),
1013        0x00000000,
1014        (0x0e00 << 16) | (0x89b4 >> 2),
1015        0x00000000,
1016        (0x0e00 << 16) | (0x3e1fc >> 2),
1017        0x00000000,
1018        (0x0e00 << 16) | (0x3c210 >> 2),
1019        0x00000000,
1020        (0x0e00 << 16) | (0x3c214 >> 2),
1021        0x00000000,
1022        (0x0e00 << 16) | (0x3c218 >> 2),
1023        0x00000000,
1024        (0x0e00 << 16) | (0x8904 >> 2),
1025        0x00000000,
1026        0x5,
1027        (0x0e00 << 16) | (0x8c28 >> 2),
1028        (0x0e00 << 16) | (0x8c2c >> 2),
1029        (0x0e00 << 16) | (0x8c30 >> 2),
1030        (0x0e00 << 16) | (0x8c34 >> 2),
1031        (0x0e00 << 16) | (0x9600 >> 2),
1032};
1033
1034static const u32 bonaire_golden_spm_registers[] =
1035{
1036        0x30800, 0xe0ffffff, 0xe0000000
1037};
1038
1039static const u32 bonaire_golden_common_registers[] =
1040{
1041        0xc770, 0xffffffff, 0x00000800,
1042        0xc774, 0xffffffff, 0x00000800,
1043        0xc798, 0xffffffff, 0x00007fbf,
1044        0xc79c, 0xffffffff, 0x00007faf
1045};
1046
1047static const u32 bonaire_golden_registers[] =
1048{
1049        0x3354, 0x00000333, 0x00000333,
1050        0x3350, 0x000c0fc0, 0x00040200,
1051        0x9a10, 0x00010000, 0x00058208,
1052        0x3c000, 0xffff1fff, 0x00140000,
1053        0x3c200, 0xfdfc0fff, 0x00000100,
1054        0x3c234, 0x40000000, 0x40000200,
1055        0x9830, 0xffffffff, 0x00000000,
1056        0x9834, 0xf00fffff, 0x00000400,
1057        0x9838, 0x0002021c, 0x00020200,
1058        0xc78, 0x00000080, 0x00000000,
1059        0x5bb0, 0x000000f0, 0x00000070,
1060        0x5bc0, 0xf0311fff, 0x80300000,
1061        0x98f8, 0x73773777, 0x12010001,
1062        0x350c, 0x00810000, 0x408af000,
1063        0x7030, 0x31000111, 0x00000011,
1064        0x2f48, 0x73773777, 0x12010001,
1065        0x220c, 0x00007fb6, 0x0021a1b1,
1066        0x2210, 0x00007fb6, 0x002021b1,
1067        0x2180, 0x00007fb6, 0x00002191,
1068        0x2218, 0x00007fb6, 0x002121b1,
1069        0x221c, 0x00007fb6, 0x002021b1,
1070        0x21dc, 0x00007fb6, 0x00002191,
1071        0x21e0, 0x00007fb6, 0x00002191,
1072        0x3628, 0x0000003f, 0x0000000a,
1073        0x362c, 0x0000003f, 0x0000000a,
1074        0x2ae4, 0x00073ffe, 0x000022a2,
1075        0x240c, 0x000007ff, 0x00000000,
1076        0x8a14, 0xf000003f, 0x00000007,
1077        0x8bf0, 0x00002001, 0x00000001,
1078        0x8b24, 0xffffffff, 0x00ffffff,
1079        0x30a04, 0x0000ff0f, 0x00000000,
1080        0x28a4c, 0x07ffffff, 0x06000000,
1081        0x4d8, 0x00000fff, 0x00000100,
1082        0x3e78, 0x00000001, 0x00000002,
1083        0x9100, 0x03000000, 0x0362c688,
1084        0x8c00, 0x000000ff, 0x00000001,
1085        0xe40, 0x00001fff, 0x00001fff,
1086        0x9060, 0x0000007f, 0x00000020,
1087        0x9508, 0x00010000, 0x00010000,
1088        0xac14, 0x000003ff, 0x000000f3,
1089        0xac0c, 0xffffffff, 0x00001032
1090};
1091
1092static const u32 bonaire_mgcg_cgcg_init[] =
1093{
1094        0xc420, 0xffffffff, 0xfffffffc,
1095        0x30800, 0xffffffff, 0xe0000000,
1096        0x3c2a0, 0xffffffff, 0x00000100,
1097        0x3c208, 0xffffffff, 0x00000100,
1098        0x3c2c0, 0xffffffff, 0xc0000100,
1099        0x3c2c8, 0xffffffff, 0xc0000100,
1100        0x3c2c4, 0xffffffff, 0xc0000100,
1101        0x55e4, 0xffffffff, 0x00600100,
1102        0x3c280, 0xffffffff, 0x00000100,
1103        0x3c214, 0xffffffff, 0x06000100,
1104        0x3c220, 0xffffffff, 0x00000100,
1105        0x3c218, 0xffffffff, 0x06000100,
1106        0x3c204, 0xffffffff, 0x00000100,
1107        0x3c2e0, 0xffffffff, 0x00000100,
1108        0x3c224, 0xffffffff, 0x00000100,
1109        0x3c200, 0xffffffff, 0x00000100,
1110        0x3c230, 0xffffffff, 0x00000100,
1111        0x3c234, 0xffffffff, 0x00000100,
1112        0x3c250, 0xffffffff, 0x00000100,
1113        0x3c254, 0xffffffff, 0x00000100,
1114        0x3c258, 0xffffffff, 0x00000100,
1115        0x3c25c, 0xffffffff, 0x00000100,
1116        0x3c260, 0xffffffff, 0x00000100,
1117        0x3c27c, 0xffffffff, 0x00000100,
1118        0x3c278, 0xffffffff, 0x00000100,
1119        0x3c210, 0xffffffff, 0x06000100,
1120        0x3c290, 0xffffffff, 0x00000100,
1121        0x3c274, 0xffffffff, 0x00000100,
1122        0x3c2b4, 0xffffffff, 0x00000100,
1123        0x3c2b0, 0xffffffff, 0x00000100,
1124        0x3c270, 0xffffffff, 0x00000100,
1125        0x30800, 0xffffffff, 0xe0000000,
1126        0x3c020, 0xffffffff, 0x00010000,
1127        0x3c024, 0xffffffff, 0x00030002,
1128        0x3c028, 0xffffffff, 0x00040007,
1129        0x3c02c, 0xffffffff, 0x00060005,
1130        0x3c030, 0xffffffff, 0x00090008,
1131        0x3c034, 0xffffffff, 0x00010000,
1132        0x3c038, 0xffffffff, 0x00030002,
1133        0x3c03c, 0xffffffff, 0x00040007,
1134        0x3c040, 0xffffffff, 0x00060005,
1135        0x3c044, 0xffffffff, 0x00090008,
1136        0x3c048, 0xffffffff, 0x00010000,
1137        0x3c04c, 0xffffffff, 0x00030002,
1138        0x3c050, 0xffffffff, 0x00040007,
1139        0x3c054, 0xffffffff, 0x00060005,
1140        0x3c058, 0xffffffff, 0x00090008,
1141        0x3c05c, 0xffffffff, 0x00010000,
1142        0x3c060, 0xffffffff, 0x00030002,
1143        0x3c064, 0xffffffff, 0x00040007,
1144        0x3c068, 0xffffffff, 0x00060005,
1145        0x3c06c, 0xffffffff, 0x00090008,
1146        0x3c070, 0xffffffff, 0x00010000,
1147        0x3c074, 0xffffffff, 0x00030002,
1148        0x3c078, 0xffffffff, 0x00040007,
1149        0x3c07c, 0xffffffff, 0x00060005,
1150        0x3c080, 0xffffffff, 0x00090008,
1151        0x3c084, 0xffffffff, 0x00010000,
1152        0x3c088, 0xffffffff, 0x00030002,
1153        0x3c08c, 0xffffffff, 0x00040007,
1154        0x3c090, 0xffffffff, 0x00060005,
1155        0x3c094, 0xffffffff, 0x00090008,
1156        0x3c098, 0xffffffff, 0x00010000,
1157        0x3c09c, 0xffffffff, 0x00030002,
1158        0x3c0a0, 0xffffffff, 0x00040007,
1159        0x3c0a4, 0xffffffff, 0x00060005,
1160        0x3c0a8, 0xffffffff, 0x00090008,
1161        0x3c000, 0xffffffff, 0x96e00200,
1162        0x8708, 0xffffffff, 0x00900100,
1163        0xc424, 0xffffffff, 0x0020003f,
1164        0x38, 0xffffffff, 0x0140001c,
1165        0x3c, 0x000f0000, 0x000f0000,
1166        0x220, 0xffffffff, 0xC060000C,
1167        0x224, 0xc0000fff, 0x00000100,
1168        0xf90, 0xffffffff, 0x00000100,
1169        0xf98, 0x00000101, 0x00000000,
1170        0x20a8, 0xffffffff, 0x00000104,
1171        0x55e4, 0xff000fff, 0x00000100,
1172        0x30cc, 0xc0000fff, 0x00000104,
1173        0xc1e4, 0x00000001, 0x00000001,
1174        0xd00c, 0xff000ff0, 0x00000100,
1175        0xd80c, 0xff000ff0, 0x00000100
1176};
1177
1178static const u32 spectre_golden_spm_registers[] =
1179{
1180        0x30800, 0xe0ffffff, 0xe0000000
1181};
1182
1183static const u32 spectre_golden_common_registers[] =
1184{
1185        0xc770, 0xffffffff, 0x00000800,
1186        0xc774, 0xffffffff, 0x00000800,
1187        0xc798, 0xffffffff, 0x00007fbf,
1188        0xc79c, 0xffffffff, 0x00007faf
1189};
1190
1191static const u32 spectre_golden_registers[] =
1192{
1193        0x3c000, 0xffff1fff, 0x96940200,
1194        0x3c00c, 0xffff0001, 0xff000000,
1195        0x3c200, 0xfffc0fff, 0x00000100,
1196        0x6ed8, 0x00010101, 0x00010000,
1197        0x9834, 0xf00fffff, 0x00000400,
1198        0x9838, 0xfffffffc, 0x00020200,
1199        0x5bb0, 0x000000f0, 0x00000070,
1200        0x5bc0, 0xf0311fff, 0x80300000,
1201        0x98f8, 0x73773777, 0x12010001,
1202        0x9b7c, 0x00ff0000, 0x00fc0000,
1203        0x2f48, 0x73773777, 0x12010001,
1204        0x8a14, 0xf000003f, 0x00000007,
1205        0x8b24, 0xffffffff, 0x00ffffff,
1206        0x28350, 0x3f3f3fff, 0x00000082,
1207        0x28354, 0x0000003f, 0x00000000,
1208        0x3e78, 0x00000001, 0x00000002,
1209        0x913c, 0xffff03df, 0x00000004,
1210        0xc768, 0x00000008, 0x00000008,
1211        0x8c00, 0x000008ff, 0x00000800,
1212        0x9508, 0x00010000, 0x00010000,
1213        0xac0c, 0xffffffff, 0x54763210,
1214        0x214f8, 0x01ff01ff, 0x00000002,
1215        0x21498, 0x007ff800, 0x00200000,
1216        0x2015c, 0xffffffff, 0x00000f40,
1217        0x30934, 0xffffffff, 0x00000001
1218};
1219
1220static const u32 spectre_mgcg_cgcg_init[] =
1221{
1222        0xc420, 0xffffffff, 0xfffffffc,
1223        0x30800, 0xffffffff, 0xe0000000,
1224        0x3c2a0, 0xffffffff, 0x00000100,
1225        0x3c208, 0xffffffff, 0x00000100,
1226        0x3c2c0, 0xffffffff, 0x00000100,
1227        0x3c2c8, 0xffffffff, 0x00000100,
1228        0x3c2c4, 0xffffffff, 0x00000100,
1229        0x55e4, 0xffffffff, 0x00600100,
1230        0x3c280, 0xffffffff, 0x00000100,
1231        0x3c214, 0xffffffff, 0x06000100,
1232        0x3c220, 0xffffffff, 0x00000100,
1233        0x3c218, 0xffffffff, 0x06000100,
1234        0x3c204, 0xffffffff, 0x00000100,
1235        0x3c2e0, 0xffffffff, 0x00000100,
1236        0x3c224, 0xffffffff, 0x00000100,
1237        0x3c200, 0xffffffff, 0x00000100,
1238        0x3c230, 0xffffffff, 0x00000100,
1239        0x3c234, 0xffffffff, 0x00000100,
1240        0x3c250, 0xffffffff, 0x00000100,
1241        0x3c254, 0xffffffff, 0x00000100,
1242        0x3c258, 0xffffffff, 0x00000100,
1243        0x3c25c, 0xffffffff, 0x00000100,
1244        0x3c260, 0xffffffff, 0x00000100,
1245        0x3c27c, 0xffffffff, 0x00000100,
1246        0x3c278, 0xffffffff, 0x00000100,
1247        0x3c210, 0xffffffff, 0x06000100,
1248        0x3c290, 0xffffffff, 0x00000100,
1249        0x3c274, 0xffffffff, 0x00000100,
1250        0x3c2b4, 0xffffffff, 0x00000100,
1251        0x3c2b0, 0xffffffff, 0x00000100,
1252        0x3c270, 0xffffffff, 0x00000100,
1253        0x30800, 0xffffffff, 0xe0000000,
1254        0x3c020, 0xffffffff, 0x00010000,
1255        0x3c024, 0xffffffff, 0x00030002,
1256        0x3c028, 0xffffffff, 0x00040007,
1257        0x3c02c, 0xffffffff, 0x00060005,
1258        0x3c030, 0xffffffff, 0x00090008,
1259        0x3c034, 0xffffffff, 0x00010000,
1260        0x3c038, 0xffffffff, 0x00030002,
1261        0x3c03c, 0xffffffff, 0x00040007,
1262        0x3c040, 0xffffffff, 0x00060005,
1263        0x3c044, 0xffffffff, 0x00090008,
1264        0x3c048, 0xffffffff, 0x00010000,
1265        0x3c04c, 0xffffffff, 0x00030002,
1266        0x3c050, 0xffffffff, 0x00040007,
1267        0x3c054, 0xffffffff, 0x00060005,
1268        0x3c058, 0xffffffff, 0x00090008,
1269        0x3c05c, 0xffffffff, 0x00010000,
1270        0x3c060, 0xffffffff, 0x00030002,
1271        0x3c064, 0xffffffff, 0x00040007,
1272        0x3c068, 0xffffffff, 0x00060005,
1273        0x3c06c, 0xffffffff, 0x00090008,
1274        0x3c070, 0xffffffff, 0x00010000,
1275        0x3c074, 0xffffffff, 0x00030002,
1276        0x3c078, 0xffffffff, 0x00040007,
1277        0x3c07c, 0xffffffff, 0x00060005,
1278        0x3c080, 0xffffffff, 0x00090008,
1279        0x3c084, 0xffffffff, 0x00010000,
1280        0x3c088, 0xffffffff, 0x00030002,
1281        0x3c08c, 0xffffffff, 0x00040007,
1282        0x3c090, 0xffffffff, 0x00060005,
1283        0x3c094, 0xffffffff, 0x00090008,
1284        0x3c098, 0xffffffff, 0x00010000,
1285        0x3c09c, 0xffffffff, 0x00030002,
1286        0x3c0a0, 0xffffffff, 0x00040007,
1287        0x3c0a4, 0xffffffff, 0x00060005,
1288        0x3c0a8, 0xffffffff, 0x00090008,
1289        0x3c0ac, 0xffffffff, 0x00010000,
1290        0x3c0b0, 0xffffffff, 0x00030002,
1291        0x3c0b4, 0xffffffff, 0x00040007,
1292        0x3c0b8, 0xffffffff, 0x00060005,
1293        0x3c0bc, 0xffffffff, 0x00090008,
1294        0x3c000, 0xffffffff, 0x96e00200,
1295        0x8708, 0xffffffff, 0x00900100,
1296        0xc424, 0xffffffff, 0x0020003f,
1297        0x38, 0xffffffff, 0x0140001c,
1298        0x3c, 0x000f0000, 0x000f0000,
1299        0x220, 0xffffffff, 0xC060000C,
1300        0x224, 0xc0000fff, 0x00000100,
1301        0xf90, 0xffffffff, 0x00000100,
1302        0xf98, 0x00000101, 0x00000000,
1303        0x20a8, 0xffffffff, 0x00000104,
1304        0x55e4, 0xff000fff, 0x00000100,
1305        0x30cc, 0xc0000fff, 0x00000104,
1306        0xc1e4, 0x00000001, 0x00000001,
1307        0xd00c, 0xff000ff0, 0x00000100,
1308        0xd80c, 0xff000ff0, 0x00000100
1309};
1310
1311static const u32 kalindi_golden_spm_registers[] =
1312{
1313        0x30800, 0xe0ffffff, 0xe0000000
1314};
1315
1316static const u32 kalindi_golden_common_registers[] =
1317{
1318        0xc770, 0xffffffff, 0x00000800,
1319        0xc774, 0xffffffff, 0x00000800,
1320        0xc798, 0xffffffff, 0x00007fbf,
1321        0xc79c, 0xffffffff, 0x00007faf
1322};
1323
1324static const u32 kalindi_golden_registers[] =
1325{
1326        0x3c000, 0xffffdfff, 0x6e944040,
1327        0x55e4, 0xff607fff, 0xfc000100,
1328        0x3c220, 0xff000fff, 0x00000100,
1329        0x3c224, 0xff000fff, 0x00000100,
1330        0x3c200, 0xfffc0fff, 0x00000100,
1331        0x6ed8, 0x00010101, 0x00010000,
1332        0x9830, 0xffffffff, 0x00000000,
1333        0x9834, 0xf00fffff, 0x00000400,
1334        0x5bb0, 0x000000f0, 0x00000070,
1335        0x5bc0, 0xf0311fff, 0x80300000,
1336        0x98f8, 0x73773777, 0x12010001,
1337        0x98fc, 0xffffffff, 0x00000010,
1338        0x9b7c, 0x00ff0000, 0x00fc0000,
1339        0x8030, 0x00001f0f, 0x0000100a,
1340        0x2f48, 0x73773777, 0x12010001,
1341        0x2408, 0x000fffff, 0x000c007f,
1342        0x8a14, 0xf000003f, 0x00000007,
1343        0x8b24, 0x3fff3fff, 0x00ffcfff,
1344        0x30a04, 0x0000ff0f, 0x00000000,
1345        0x28a4c, 0x07ffffff, 0x06000000,
1346        0x4d8, 0x00000fff, 0x00000100,
1347        0x3e78, 0x00000001, 0x00000002,
1348        0xc768, 0x00000008, 0x00000008,
1349        0x8c00, 0x000000ff, 0x00000003,
1350        0x214f8, 0x01ff01ff, 0x00000002,
1351        0x21498, 0x007ff800, 0x00200000,
1352        0x2015c, 0xffffffff, 0x00000f40,
1353        0x88c4, 0x001f3ae3, 0x00000082,
1354        0x88d4, 0x0000001f, 0x00000010,
1355        0x30934, 0xffffffff, 0x00000000
1356};
1357
1358static const u32 kalindi_mgcg_cgcg_init[] =
1359{
1360        0xc420, 0xffffffff, 0xfffffffc,
1361        0x30800, 0xffffffff, 0xe0000000,
1362        0x3c2a0, 0xffffffff, 0x00000100,
1363        0x3c208, 0xffffffff, 0x00000100,
1364        0x3c2c0, 0xffffffff, 0x00000100,
1365        0x3c2c8, 0xffffffff, 0x00000100,
1366        0x3c2c4, 0xffffffff, 0x00000100,
1367        0x55e4, 0xffffffff, 0x00600100,
1368        0x3c280, 0xffffffff, 0x00000100,
1369        0x3c214, 0xffffffff, 0x06000100,
1370        0x3c220, 0xffffffff, 0x00000100,
1371        0x3c218, 0xffffffff, 0x06000100,
1372        0x3c204, 0xffffffff, 0x00000100,
1373        0x3c2e0, 0xffffffff, 0x00000100,
1374        0x3c224, 0xffffffff, 0x00000100,
1375        0x3c200, 0xffffffff, 0x00000100,
1376        0x3c230, 0xffffffff, 0x00000100,
1377        0x3c234, 0xffffffff, 0x00000100,
1378        0x3c250, 0xffffffff, 0x00000100,
1379        0x3c254, 0xffffffff, 0x00000100,
1380        0x3c258, 0xffffffff, 0x00000100,
1381        0x3c25c, 0xffffffff, 0x00000100,
1382        0x3c260, 0xffffffff, 0x00000100,
1383        0x3c27c, 0xffffffff, 0x00000100,
1384        0x3c278, 0xffffffff, 0x00000100,
1385        0x3c210, 0xffffffff, 0x06000100,
1386        0x3c290, 0xffffffff, 0x00000100,
1387        0x3c274, 0xffffffff, 0x00000100,
1388        0x3c2b4, 0xffffffff, 0x00000100,
1389        0x3c2b0, 0xffffffff, 0x00000100,
1390        0x3c270, 0xffffffff, 0x00000100,
1391        0x30800, 0xffffffff, 0xe0000000,
1392        0x3c020, 0xffffffff, 0x00010000,
1393        0x3c024, 0xffffffff, 0x00030002,
1394        0x3c028, 0xffffffff, 0x00040007,
1395        0x3c02c, 0xffffffff, 0x00060005,
1396        0x3c030, 0xffffffff, 0x00090008,
1397        0x3c034, 0xffffffff, 0x00010000,
1398        0x3c038, 0xffffffff, 0x00030002,
1399        0x3c03c, 0xffffffff, 0x00040007,
1400        0x3c040, 0xffffffff, 0x00060005,
1401        0x3c044, 0xffffffff, 0x00090008,
1402        0x3c000, 0xffffffff, 0x96e00200,
1403        0x8708, 0xffffffff, 0x00900100,
1404        0xc424, 0xffffffff, 0x0020003f,
1405        0x38, 0xffffffff, 0x0140001c,
1406        0x3c, 0x000f0000, 0x000f0000,
1407        0x220, 0xffffffff, 0xC060000C,
1408        0x224, 0xc0000fff, 0x00000100,
1409        0x20a8, 0xffffffff, 0x00000104,
1410        0x55e4, 0xff000fff, 0x00000100,
1411        0x30cc, 0xc0000fff, 0x00000104,
1412        0xc1e4, 0x00000001, 0x00000001,
1413        0xd00c, 0xff000ff0, 0x00000100,
1414        0xd80c, 0xff000ff0, 0x00000100
1415};
1416
1417static const u32 hawaii_golden_spm_registers[] =
1418{
1419        0x30800, 0xe0ffffff, 0xe0000000
1420};
1421
1422static const u32 hawaii_golden_common_registers[] =
1423{
1424        0x30800, 0xffffffff, 0xe0000000,
1425        0x28350, 0xffffffff, 0x3a00161a,
1426        0x28354, 0xffffffff, 0x0000002e,
1427        0x9a10, 0xffffffff, 0x00018208,
1428        0x98f8, 0xffffffff, 0x12011003
1429};
1430
1431static const u32 hawaii_golden_registers[] =
1432{
1433        0x3354, 0x00000333, 0x00000333,
1434        0x9a10, 0x00010000, 0x00058208,
1435        0x9830, 0xffffffff, 0x00000000,
1436        0x9834, 0xf00fffff, 0x00000400,
1437        0x9838, 0x0002021c, 0x00020200,
1438        0xc78, 0x00000080, 0x00000000,
1439        0x5bb0, 0x000000f0, 0x00000070,
1440        0x5bc0, 0xf0311fff, 0x80300000,
1441        0x350c, 0x00810000, 0x408af000,
1442        0x7030, 0x31000111, 0x00000011,
1443        0x2f48, 0x73773777, 0x12010001,
1444        0x2120, 0x0000007f, 0x0000001b,
1445        0x21dc, 0x00007fb6, 0x00002191,
1446        0x3628, 0x0000003f, 0x0000000a,
1447        0x362c, 0x0000003f, 0x0000000a,
1448        0x2ae4, 0x00073ffe, 0x000022a2,
1449        0x240c, 0x000007ff, 0x00000000,
1450        0x8bf0, 0x00002001, 0x00000001,
1451        0x8b24, 0xffffffff, 0x00ffffff,
1452        0x30a04, 0x0000ff0f, 0x00000000,
1453        0x28a4c, 0x07ffffff, 0x06000000,
1454        0x3e78, 0x00000001, 0x00000002,
1455        0xc768, 0x00000008, 0x00000008,
1456        0xc770, 0x00000f00, 0x00000800,
1457        0xc774, 0x00000f00, 0x00000800,
1458        0xc798, 0x00ffffff, 0x00ff7fbf,
1459        0xc79c, 0x00ffffff, 0x00ff7faf,
1460        0x8c00, 0x000000ff, 0x00000800,
1461        0xe40, 0x00001fff, 0x00001fff,
1462        0x9060, 0x0000007f, 0x00000020,
1463        0x9508, 0x00010000, 0x00010000,
1464        0xae00, 0x00100000, 0x000ff07c,
1465        0xac14, 0x000003ff, 0x0000000f,
1466        0xac10, 0xffffffff, 0x7564fdec,
1467        0xac0c, 0xffffffff, 0x3120b9a8,
1468        0xac08, 0x20000000, 0x0f9c0000
1469};
1470
1471static const u32 hawaii_mgcg_cgcg_init[] =
1472{
1473        0xc420, 0xffffffff, 0xfffffffd,
1474        0x30800, 0xffffffff, 0xe0000000,
1475        0x3c2a0, 0xffffffff, 0x00000100,
1476        0x3c208, 0xffffffff, 0x00000100,
1477        0x3c2c0, 0xffffffff, 0x00000100,
1478        0x3c2c8, 0xffffffff, 0x00000100,
1479        0x3c2c4, 0xffffffff, 0x00000100,
1480        0x55e4, 0xffffffff, 0x00200100,
1481        0x3c280, 0xffffffff, 0x00000100,
1482        0x3c214, 0xffffffff, 0x06000100,
1483        0x3c220, 0xffffffff, 0x00000100,
1484        0x3c218, 0xffffffff, 0x06000100,
1485        0x3c204, 0xffffffff, 0x00000100,
1486        0x3c2e0, 0xffffffff, 0x00000100,
1487        0x3c224, 0xffffffff, 0x00000100,
1488        0x3c200, 0xffffffff, 0x00000100,
1489        0x3c230, 0xffffffff, 0x00000100,
1490        0x3c234, 0xffffffff, 0x00000100,
1491        0x3c250, 0xffffffff, 0x00000100,
1492        0x3c254, 0xffffffff, 0x00000100,
1493        0x3c258, 0xffffffff, 0x00000100,
1494        0x3c25c, 0xffffffff, 0x00000100,
1495        0x3c260, 0xffffffff, 0x00000100,
1496        0x3c27c, 0xffffffff, 0x00000100,
1497        0x3c278, 0xffffffff, 0x00000100,
1498        0x3c210, 0xffffffff, 0x06000100,
1499        0x3c290, 0xffffffff, 0x00000100,
1500        0x3c274, 0xffffffff, 0x00000100,
1501        0x3c2b4, 0xffffffff, 0x00000100,
1502        0x3c2b0, 0xffffffff, 0x00000100,
1503        0x3c270, 0xffffffff, 0x00000100,
1504        0x30800, 0xffffffff, 0xe0000000,
1505        0x3c020, 0xffffffff, 0x00010000,
1506        0x3c024, 0xffffffff, 0x00030002,
1507        0x3c028, 0xffffffff, 0x00040007,
1508        0x3c02c, 0xffffffff, 0x00060005,
1509        0x3c030, 0xffffffff, 0x00090008,
1510        0x3c034, 0xffffffff, 0x00010000,
1511        0x3c038, 0xffffffff, 0x00030002,
1512        0x3c03c, 0xffffffff, 0x00040007,
1513        0x3c040, 0xffffffff, 0x00060005,
1514        0x3c044, 0xffffffff, 0x00090008,
1515        0x3c048, 0xffffffff, 0x00010000,
1516        0x3c04c, 0xffffffff, 0x00030002,
1517        0x3c050, 0xffffffff, 0x00040007,
1518        0x3c054, 0xffffffff, 0x00060005,
1519        0x3c058, 0xffffffff, 0x00090008,
1520        0x3c05c, 0xffffffff, 0x00010000,
1521        0x3c060, 0xffffffff, 0x00030002,
1522        0x3c064, 0xffffffff, 0x00040007,
1523        0x3c068, 0xffffffff, 0x00060005,
1524        0x3c06c, 0xffffffff, 0x00090008,
1525        0x3c070, 0xffffffff, 0x00010000,
1526        0x3c074, 0xffffffff, 0x00030002,
1527        0x3c078, 0xffffffff, 0x00040007,
1528        0x3c07c, 0xffffffff, 0x00060005,
1529        0x3c080, 0xffffffff, 0x00090008,
1530        0x3c084, 0xffffffff, 0x00010000,
1531        0x3c088, 0xffffffff, 0x00030002,
1532        0x3c08c, 0xffffffff, 0x00040007,
1533        0x3c090, 0xffffffff, 0x00060005,
1534        0x3c094, 0xffffffff, 0x00090008,
1535        0x3c098, 0xffffffff, 0x00010000,
1536        0x3c09c, 0xffffffff, 0x00030002,
1537        0x3c0a0, 0xffffffff, 0x00040007,
1538        0x3c0a4, 0xffffffff, 0x00060005,
1539        0x3c0a8, 0xffffffff, 0x00090008,
1540        0x3c0ac, 0xffffffff, 0x00010000,
1541        0x3c0b0, 0xffffffff, 0x00030002,
1542        0x3c0b4, 0xffffffff, 0x00040007,
1543        0x3c0b8, 0xffffffff, 0x00060005,
1544        0x3c0bc, 0xffffffff, 0x00090008,
1545        0x3c0c0, 0xffffffff, 0x00010000,
1546        0x3c0c4, 0xffffffff, 0x00030002,
1547        0x3c0c8, 0xffffffff, 0x00040007,
1548        0x3c0cc, 0xffffffff, 0x00060005,
1549        0x3c0d0, 0xffffffff, 0x00090008,
1550        0x3c0d4, 0xffffffff, 0x00010000,
1551        0x3c0d8, 0xffffffff, 0x00030002,
1552        0x3c0dc, 0xffffffff, 0x00040007,
1553        0x3c0e0, 0xffffffff, 0x00060005,
1554        0x3c0e4, 0xffffffff, 0x00090008,
1555        0x3c0e8, 0xffffffff, 0x00010000,
1556        0x3c0ec, 0xffffffff, 0x00030002,
1557        0x3c0f0, 0xffffffff, 0x00040007,
1558        0x3c0f4, 0xffffffff, 0x00060005,
1559        0x3c0f8, 0xffffffff, 0x00090008,
1560        0xc318, 0xffffffff, 0x00020200,
1561        0x3350, 0xffffffff, 0x00000200,
1562        0x15c0, 0xffffffff, 0x00000400,
1563        0x55e8, 0xffffffff, 0x00000000,
1564        0x2f50, 0xffffffff, 0x00000902,
1565        0x3c000, 0xffffffff, 0x96940200,
1566        0x8708, 0xffffffff, 0x00900100,
1567        0xc424, 0xffffffff, 0x0020003f,
1568        0x38, 0xffffffff, 0x0140001c,
1569        0x3c, 0x000f0000, 0x000f0000,
1570        0x220, 0xffffffff, 0xc060000c,
1571        0x224, 0xc0000fff, 0x00000100,
1572        0xf90, 0xffffffff, 0x00000100,
1573        0xf98, 0x00000101, 0x00000000,
1574        0x20a8, 0xffffffff, 0x00000104,
1575        0x55e4, 0xff000fff, 0x00000100,
1576        0x30cc, 0xc0000fff, 0x00000104,
1577        0xc1e4, 0x00000001, 0x00000001,
1578        0xd00c, 0xff000ff0, 0x00000100,
1579        0xd80c, 0xff000ff0, 0x00000100
1580};
1581
1582static const u32 godavari_golden_registers[] =
1583{
1584        0x55e4, 0xff607fff, 0xfc000100,
1585        0x6ed8, 0x00010101, 0x00010000,
1586        0x9830, 0xffffffff, 0x00000000,
1587        0x98302, 0xf00fffff, 0x00000400,
1588        0x6130, 0xffffffff, 0x00010000,
1589        0x5bb0, 0x000000f0, 0x00000070,
1590        0x5bc0, 0xf0311fff, 0x80300000,
1591        0x98f8, 0x73773777, 0x12010001,
1592        0x98fc, 0xffffffff, 0x00000010,
1593        0x8030, 0x00001f0f, 0x0000100a,
1594        0x2f48, 0x73773777, 0x12010001,
1595        0x2408, 0x000fffff, 0x000c007f,
1596        0x8a14, 0xf000003f, 0x00000007,
1597        0x8b24, 0xffffffff, 0x00ff0fff,
1598        0x30a04, 0x0000ff0f, 0x00000000,
1599        0x28a4c, 0x07ffffff, 0x06000000,
1600        0x4d8, 0x00000fff, 0x00000100,
1601        0xd014, 0x00010000, 0x00810001,
1602        0xd814, 0x00010000, 0x00810001,
1603        0x3e78, 0x00000001, 0x00000002,
1604        0xc768, 0x00000008, 0x00000008,
1605        0xc770, 0x00000f00, 0x00000800,
1606        0xc774, 0x00000f00, 0x00000800,
1607        0xc798, 0x00ffffff, 0x00ff7fbf,
1608        0xc79c, 0x00ffffff, 0x00ff7faf,
1609        0x8c00, 0x000000ff, 0x00000001,
1610        0x214f8, 0x01ff01ff, 0x00000002,
1611        0x21498, 0x007ff800, 0x00200000,
1612        0x2015c, 0xffffffff, 0x00000f40,
1613        0x88c4, 0x001f3ae3, 0x00000082,
1614        0x88d4, 0x0000001f, 0x00000010,
1615        0x30934, 0xffffffff, 0x00000000
1616};
1617
1618
1619static void cik_init_golden_registers(struct radeon_device *rdev)
1620{
1621        switch (rdev->family) {
1622        case CHIP_BONAIRE:
1623                radeon_program_register_sequence(rdev,
1624                                                 bonaire_mgcg_cgcg_init,
1625                                                 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1626                radeon_program_register_sequence(rdev,
1627                                                 bonaire_golden_registers,
1628                                                 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1629                radeon_program_register_sequence(rdev,
1630                                                 bonaire_golden_common_registers,
1631                                                 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1632                radeon_program_register_sequence(rdev,
1633                                                 bonaire_golden_spm_registers,
1634                                                 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1635                break;
1636        case CHIP_KABINI:
1637                radeon_program_register_sequence(rdev,
1638                                                 kalindi_mgcg_cgcg_init,
1639                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1640                radeon_program_register_sequence(rdev,
1641                                                 kalindi_golden_registers,
1642                                                 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1643                radeon_program_register_sequence(rdev,
1644                                                 kalindi_golden_common_registers,
1645                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1646                radeon_program_register_sequence(rdev,
1647                                                 kalindi_golden_spm_registers,
1648                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1649                break;
1650        case CHIP_MULLINS:
1651                radeon_program_register_sequence(rdev,
1652                                                 kalindi_mgcg_cgcg_init,
1653                                                 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1654                radeon_program_register_sequence(rdev,
1655                                                 godavari_golden_registers,
1656                                                 (const u32)ARRAY_SIZE(godavari_golden_registers));
1657                radeon_program_register_sequence(rdev,
1658                                                 kalindi_golden_common_registers,
1659                                                 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1660                radeon_program_register_sequence(rdev,
1661                                                 kalindi_golden_spm_registers,
1662                                                 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1663                break;
1664        case CHIP_KAVERI:
1665                radeon_program_register_sequence(rdev,
1666                                                 spectre_mgcg_cgcg_init,
1667                                                 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1668                radeon_program_register_sequence(rdev,
1669                                                 spectre_golden_registers,
1670                                                 (const u32)ARRAY_SIZE(spectre_golden_registers));
1671                radeon_program_register_sequence(rdev,
1672                                                 spectre_golden_common_registers,
1673                                                 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1674                radeon_program_register_sequence(rdev,
1675                                                 spectre_golden_spm_registers,
1676                                                 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1677                break;
1678        case CHIP_HAWAII:
1679                radeon_program_register_sequence(rdev,
1680                                                 hawaii_mgcg_cgcg_init,
1681                                                 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1682                radeon_program_register_sequence(rdev,
1683                                                 hawaii_golden_registers,
1684                                                 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1685                radeon_program_register_sequence(rdev,
1686                                                 hawaii_golden_common_registers,
1687                                                 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1688                radeon_program_register_sequence(rdev,
1689                                                 hawaii_golden_spm_registers,
1690                                                 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1691                break;
1692        default:
1693                break;
1694        }
1695}
1696
1697/**
1698 * cik_get_xclk - get the xclk
1699 *
1700 * @rdev: radeon_device pointer
1701 *
1702 * Returns the reference clock used by the gfx engine
1703 * (CIK).
1704 */
1705u32 cik_get_xclk(struct radeon_device *rdev)
1706{
1707        u32 reference_clock = rdev->clock.spll.reference_freq;
1708
1709        if (rdev->flags & RADEON_IS_IGP) {
1710                if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1711                        return reference_clock / 2;
1712        } else {
1713                if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1714                        return reference_clock / 4;
1715        }
1716        return reference_clock;
1717}
1718
1719/**
1720 * cik_mm_rdoorbell - read a doorbell dword
1721 *
1722 * @rdev: radeon_device pointer
1723 * @index: doorbell index
1724 *
1725 * Returns the value in the doorbell aperture at the
1726 * requested doorbell index (CIK).
1727 */
1728u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1729{
1730        if (index < rdev->doorbell.num_doorbells) {
1731                return readl(rdev->doorbell.ptr + index);
1732        } else {
1733                DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1734                return 0;
1735        }
1736}
1737
1738/**
1739 * cik_mm_wdoorbell - write a doorbell dword
1740 *
1741 * @rdev: radeon_device pointer
1742 * @index: doorbell index
1743 * @v: value to write
1744 *
1745 * Writes @v to the doorbell aperture at the
1746 * requested doorbell index (CIK).
1747 */
1748void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1749{
1750        if (index < rdev->doorbell.num_doorbells) {
1751                writel(v, rdev->doorbell.ptr + index);
1752        } else {
1753                DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1754        }
1755}
1756
1757#define BONAIRE_IO_MC_REGS_SIZE 36
1758
1759static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1760{
1761        {0x00000070, 0x04400000},
1762        {0x00000071, 0x80c01803},
1763        {0x00000072, 0x00004004},
1764        {0x00000073, 0x00000100},
1765        {0x00000074, 0x00ff0000},
1766        {0x00000075, 0x34000000},
1767        {0x00000076, 0x08000014},
1768        {0x00000077, 0x00cc08ec},
1769        {0x00000078, 0x00000400},
1770        {0x00000079, 0x00000000},
1771        {0x0000007a, 0x04090000},
1772        {0x0000007c, 0x00000000},
1773        {0x0000007e, 0x4408a8e8},
1774        {0x0000007f, 0x00000304},
1775        {0x00000080, 0x00000000},
1776        {0x00000082, 0x00000001},
1777        {0x00000083, 0x00000002},
1778        {0x00000084, 0xf3e4f400},
1779        {0x00000085, 0x052024e3},
1780        {0x00000087, 0x00000000},
1781        {0x00000088, 0x01000000},
1782        {0x0000008a, 0x1c0a0000},
1783        {0x0000008b, 0xff010000},
1784        {0x0000008d, 0xffffefff},
1785        {0x0000008e, 0xfff3efff},
1786        {0x0000008f, 0xfff3efbf},
1787        {0x00000092, 0xf7ffffff},
1788        {0x00000093, 0xffffff7f},
1789        {0x00000095, 0x00101101},
1790        {0x00000096, 0x00000fff},
1791        {0x00000097, 0x00116fff},
1792        {0x00000098, 0x60010000},
1793        {0x00000099, 0x10010000},
1794        {0x0000009a, 0x00006000},
1795        {0x0000009b, 0x00001000},
1796        {0x0000009f, 0x00b48000}
1797};
1798
1799#define HAWAII_IO_MC_REGS_SIZE 22
1800
1801static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1802{
1803        {0x0000007d, 0x40000000},
1804        {0x0000007e, 0x40180304},
1805        {0x0000007f, 0x0000ff00},
1806        {0x00000081, 0x00000000},
1807        {0x00000083, 0x00000800},
1808        {0x00000086, 0x00000000},
1809        {0x00000087, 0x00000100},
1810        {0x00000088, 0x00020100},
1811        {0x00000089, 0x00000000},
1812        {0x0000008b, 0x00040000},
1813        {0x0000008c, 0x00000100},
1814        {0x0000008e, 0xff010000},
1815        {0x00000090, 0xffffefff},
1816        {0x00000091, 0xfff3efff},
1817        {0x00000092, 0xfff3efbf},
1818        {0x00000093, 0xf7ffffff},
1819        {0x00000094, 0xffffff7f},
1820        {0x00000095, 0x00000fff},
1821        {0x00000096, 0x00116fff},
1822        {0x00000097, 0x60010000},
1823        {0x00000098, 0x10010000},
1824        {0x0000009f, 0x00c79000}
1825};
1826
1827
1828/**
1829 * cik_srbm_select - select specific register instances
1830 *
1831 * @rdev: radeon_device pointer
1832 * @me: selected ME (micro engine)
1833 * @pipe: pipe
1834 * @queue: queue
1835 * @vmid: VMID
1836 *
1837 * Switches the currently active registers instances.  Some
1838 * registers are instanced per VMID, others are instanced per
1839 * me/pipe/queue combination.
1840 */
1841static void cik_srbm_select(struct radeon_device *rdev,
1842                            u32 me, u32 pipe, u32 queue, u32 vmid)
1843{
1844        u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1845                             MEID(me & 0x3) |
1846                             VMID(vmid & 0xf) |
1847                             QUEUEID(queue & 0x7));
1848        WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1849}
1850
1851/* ucode loading */
1852/**
1853 * ci_mc_load_microcode - load MC ucode into the hw
1854 *
1855 * @rdev: radeon_device pointer
1856 *
1857 * Load the GDDR MC ucode into the hw (CIK).
1858 * Returns 0 on success, error on failure.
1859 */
1860int ci_mc_load_microcode(struct radeon_device *rdev)
1861{
1862        const __be32 *fw_data = NULL;
1863        const __le32 *new_fw_data = NULL;
1864        u32 running, tmp;
1865        u32 *io_mc_regs = NULL;
1866        const __le32 *new_io_mc_regs = NULL;
1867        int i, regs_size, ucode_size;
1868
1869        if (!rdev->mc_fw)
1870                return -EINVAL;
1871
1872        if (rdev->new_fw) {
1873                const struct mc_firmware_header_v1_0 *hdr =
1874                        (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1875
1876                radeon_ucode_print_mc_hdr(&hdr->header);
1877
1878                regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1879                new_io_mc_regs = (const __le32 *)
1880                        (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1881                ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1882                new_fw_data = (const __le32 *)
1883                        (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1884        } else {
1885                ucode_size = rdev->mc_fw->size / 4;
1886
1887                switch (rdev->family) {
1888                case CHIP_BONAIRE:
1889                        io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1890                        regs_size = BONAIRE_IO_MC_REGS_SIZE;
1891                        break;
1892                case CHIP_HAWAII:
1893                        io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1894                        regs_size = HAWAII_IO_MC_REGS_SIZE;
1895                        break;
1896                default:
1897                        return -EINVAL;
1898                }
1899                fw_data = (const __be32 *)rdev->mc_fw->data;
1900        }
1901
1902        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1903
1904        if (running == 0) {
1905                /* reset the engine and set to writable */
1906                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1907                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1908
1909                /* load mc io regs */
1910                for (i = 0; i < regs_size; i++) {
1911                        if (rdev->new_fw) {
1912                                WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1913                                WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1914                        } else {
1915                                WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1916                                WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1917                        }
1918                }
1919
1920                tmp = RREG32(MC_SEQ_MISC0);
1921                if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1922                        WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1923                        WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1924                        WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1925                        WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1926                }
1927
1928                /* load the MC ucode */
1929                for (i = 0; i < ucode_size; i++) {
1930                        if (rdev->new_fw)
1931                                WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1932                        else
1933                                WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1934                }
1935
1936                /* put the engine back into the active state */
1937                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1938                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1939                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1940
1941                /* wait for training to complete */
1942                for (i = 0; i < rdev->usec_timeout; i++) {
1943                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1944                                break;
1945                        udelay(1);
1946                }
1947                for (i = 0; i < rdev->usec_timeout; i++) {
1948                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1949                                break;
1950                        udelay(1);
1951                }
1952        }
1953
1954        return 0;
1955}
1956
1957/**
1958 * cik_init_microcode - load ucode images from disk
1959 *
1960 * @rdev: radeon_device pointer
1961 *
1962 * Use the firmware interface to load the ucode images into
1963 * the driver (not loaded into hw).
1964 * Returns 0 on success, error on failure.
1965 */
1966static int cik_init_microcode(struct radeon_device *rdev)
1967{
1968        const char *chip_name;
1969        const char *new_chip_name;
1970        size_t pfp_req_size, me_req_size, ce_req_size,
1971                mec_req_size, rlc_req_size, mc_req_size = 0,
1972                sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1973        char fw_name[30];
1974        int new_fw = 0;
1975        int err;
1976        int num_fw;
1977        bool new_smc = false;
1978
1979        DRM_DEBUG("\n");
1980
1981        switch (rdev->family) {
1982        case CHIP_BONAIRE:
1983                chip_name = "BONAIRE";
1984                if ((rdev->pdev->revision == 0x80) ||
1985                    (rdev->pdev->revision == 0x81) ||
1986                    (rdev->pdev->device == 0x665f))
1987                        new_smc = true;
1988                new_chip_name = "bonaire";
1989                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1990                me_req_size = CIK_ME_UCODE_SIZE * 4;
1991                ce_req_size = CIK_CE_UCODE_SIZE * 4;
1992                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1993                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1994                mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1995                mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1996                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1997                smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1998                num_fw = 8;
1999                break;
2000        case CHIP_HAWAII:

2001                chip_name = "HAWAII";
2002                if (rdev->pdev->revision == 0x80)
2003                        new_smc = true;
2004                new_chip_name = "hawaii";
2005                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2006                me_req_size = CIK_ME_UCODE_SIZE * 4;
2007                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2008                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2009                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2010                mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2011                mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2012                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2013                smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2014                num_fw = 8;
2015                break;
2016        case CHIP_KAVERI:
2017                chip_name = "KAVERI";
2018                new_chip_name = "kaveri";
2019                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2020                me_req_size = CIK_ME_UCODE_SIZE * 4;
2021                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2022                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2023                rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2024                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2025                num_fw = 7;
2026                break;
2027        case CHIP_KABINI:
2028                chip_name = "KABINI";
2029                new_chip_name = "kabini";
2030                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2031                me_req_size = CIK_ME_UCODE_SIZE * 4;
2032                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2033                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2034                rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2035                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2036                num_fw = 6;
2037                break;
2038        case CHIP_MULLINS:
2039                chip_name = "MULLINS";
2040                new_chip_name = "mullins";
2041                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2042                me_req_size = CIK_ME_UCODE_SIZE * 4;
2043                ce_req_size = CIK_CE_UCODE_SIZE * 4;
2044                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2045                rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2046                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2047                num_fw = 6;
2048                break;
2049        default: BUG();
2050        }
2051
2052        DRM_INFO("Loading %s Microcode\n", new_chip_name);
2053
2054        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
2055        err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2056        if (err) {
2057                snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2058                err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2059                if (err)
2060                        goto out;
2061                if (rdev->pfp_fw->size != pfp_req_size) {
2062                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2063                               rdev->pfp_fw->size, fw_name);
2064                        err = -EINVAL;
2065                        goto out;
2066                }
2067        } else {
2068                err = radeon_ucode_validate(rdev->pfp_fw);
2069                if (err) {
2070                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2071                               fw_name);
2072                        goto out;
2073                } else {
2074                        new_fw++;
2075                }
2076        }
2077
2078        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2079        err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2080        if (err) {
2081                snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2082                err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2083                if (err)
2084                        goto out;
2085                if (rdev->me_fw->size != me_req_size) {
2086                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2087                               rdev->me_fw->size, fw_name);
2088                        err = -EINVAL;
2089                }
2090        } else {
2091                err = radeon_ucode_validate(rdev->me_fw);
2092                if (err) {
2093                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2094                               fw_name);
2095                        goto out;
2096                } else {
2097                        new_fw++;
2098                }
2099        }
2100
2101        snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2102        err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2103        if (err) {
2104                snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2105                err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2106                if (err)
2107                        goto out;
2108                if (rdev->ce_fw->size != ce_req_size) {
2109                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2110                               rdev->ce_fw->size, fw_name);
2111                        err = -EINVAL;
2112                }
2113        } else {
2114                err = radeon_ucode_validate(rdev->ce_fw);
2115                if (err) {
2116                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2117                               fw_name);
2118                        goto out;
2119                } else {
2120                        new_fw++;
2121                }
2122        }
2123
2124        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2125        err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2126        if (err) {
2127                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2128                err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2129                if (err)
2130                        goto out;
2131                if (rdev->mec_fw->size != mec_req_size) {
2132                        pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2133                               rdev->mec_fw->size, fw_name);
2134                        err = -EINVAL;
2135                }
2136        } else {
2137                err = radeon_ucode_validate(rdev->mec_fw);
2138                if (err) {
2139                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2140                               fw_name);
2141                        goto out;
2142                } else {
2143                        new_fw++;
2144                }
2145        }
2146
2147        if (rdev->family == CHIP_KAVERI) {
2148                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2149                err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2150                if (err) {
2151                        goto out;
2152                } else {
2153                        err = radeon_ucode_validate(rdev->mec2_fw);
2154                        if (err) {
2155                                goto out;
2156                        } else {
2157                                new_fw++;
2158                        }
2159                }
2160        }
2161
2162        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2163        err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2164        if (err) {
2165                snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2166                err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2167                if (err)
2168                        goto out;
2169                if (rdev->rlc_fw->size != rlc_req_size) {
2170                        pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2171                               rdev->rlc_fw->size, fw_name);
2172                        err = -EINVAL;
2173                }
2174        } else {
2175                err = radeon_ucode_validate(rdev->rlc_fw);
2176                if (err) {
2177                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2178                               fw_name);
2179                        goto out;
2180                } else {
2181                        new_fw++;
2182                }
2183        }
2184
2185        snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2186        err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2187        if (err) {
2188                snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2189                err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2190                if (err)
2191                        goto out;
2192                if (rdev->sdma_fw->size != sdma_req_size) {
2193                        pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2194                               rdev->sdma_fw->size, fw_name);
2195                        err = -EINVAL;
2196                }
2197        } else {
2198                err = radeon_ucode_validate(rdev->sdma_fw);
2199                if (err) {
2200                        pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2201                               fw_name);
2202                        goto out;
2203                } else {
2204                        new_fw++;
2205                }
2206        }
2207
2208        /* No SMC, MC ucode on APUs */
2209        if (!(rdev->flags & RADEON_IS_IGP)) {
2210                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2211                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2212                if (err) {
2213                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2214                        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2215                        if (err) {
2216                                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2217                                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2218                                if (err)
2219                                        goto out;
2220                        }
2221                        if ((rdev->mc_fw->size != mc_req_size) &&
2222                            (rdev->mc_fw->size != mc2_req_size)){
2223                                pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
2224                                       rdev->mc_fw->size, fw_name);
2225                                err = -EINVAL;
2226                        }
2227                        DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2228                } else {
2229                        err = radeon_ucode_validate(rdev->mc_fw);
2230                        if (err) {
2231                                pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2232                                       fw_name);
2233                                goto out;
2234                        } else {
2235                                new_fw++;
2236                        }
2237                }
2238
2239                if (new_smc)
2240                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2241                else
2242                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2243                err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2244                if (err) {
2245                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2246                        err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2247                        if (err) {
2248                                pr_err("smc: error loading firmware \"%s\"\n",
2249                                       fw_name);
2250                                release_firmware(rdev->smc_fw);
2251                                rdev->smc_fw = NULL;
2252                                err = 0;
2253                        } else if (rdev->smc_fw->size != smc_req_size) {
2254                                pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
2255                                       rdev->smc_fw->size, fw_name);
2256                                err = -EINVAL;
2257                        }
2258                } else {
2259                        err = radeon_ucode_validate(rdev->smc_fw);
2260                        if (err) {
2261                                pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2262                                       fw_name);
2263                                goto out;
2264                        } else {
2265                                new_fw++;
2266                        }
2267                }
2268        }
2269
2270        if (new_fw == 0) {
2271                rdev->new_fw = false;
2272        } else if (new_fw < num_fw) {
2273                pr_err("ci_fw: mixing new and old firmware!\n");
2274                err = -EINVAL;
2275        } else {
2276                rdev->new_fw = true;
2277        }
2278
2279out:
2280        if (err) {
2281                if (err != -EINVAL)
2282                        pr_err("cik_cp: Failed to load firmware \"%s\"\n",
2283                               fw_name);
2284                release_firmware(rdev->pfp_fw);
2285                rdev->pfp_fw = NULL;
2286                release_firmware(rdev->me_fw);
2287                rdev->me_fw = NULL;
2288                release_firmware(rdev->ce_fw);
2289                rdev->ce_fw = NULL;
2290                release_firmware(rdev->mec_fw);
2291                rdev->mec_fw = NULL;
2292                release_firmware(rdev->mec2_fw);
2293                rdev->mec2_fw = NULL;
2294                release_firmware(rdev->rlc_fw);
2295                rdev->rlc_fw = NULL;
2296                release_firmware(rdev->sdma_fw);
2297                rdev->sdma_fw = NULL;
2298                release_firmware(rdev->mc_fw);
2299                rdev->mc_fw = NULL;
2300                release_firmware(rdev->smc_fw);
2301                rdev->smc_fw = NULL;
2302        }
2303        return err;
2304}
2305
2306/*
2307 * Core functions
2308 */
2309/**
2310 * cik_tiling_mode_table_init - init the hw tiling table
2311 *
2312 * @rdev: radeon_device pointer
2313 *
2314 * Starting with SI, the tiling setup is done globally in a
2315 * set of 32 tiling modes.  Rather than selecting each set of
2316 * parameters per surface as on older asics, we just select
2317 * which index in the tiling table we want to use, and the
2318 * surface uses those parameters (CIK).
2319 */
2320static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2321{
2322        u32 *tile = rdev->config.cik.tile_mode_array;
2323        u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2324        const u32 num_tile_mode_states =
2325                        ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2326        const u32 num_secondary_tile_mode_states =
2327                        ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2328        u32 reg_offset, split_equal_to_row_size;
2329        u32 num_pipe_configs;
2330        u32 num_rbs = rdev->config.cik.max_backends_per_se *
2331                rdev->config.cik.max_shader_engines;
2332
2333        switch (rdev->config.cik.mem_row_size_in_kb) {
2334        case 1:
2335                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2336                break;
2337        case 2:
2338        default:
2339                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2340                break;
2341        case 4:
2342                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2343                break;
2344        }
2345
2346        num_pipe_configs = rdev->config.cik.max_tile_pipes;
2347        if (num_pipe_configs > 8)
2348                num_pipe_configs = 16;
2349
2350        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2351                tile[reg_offset] = 0;
2352        for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2353                macrotile[reg_offset] = 0;
2354
2355        switch(num_pipe_configs) {
2356        case 16:
2357                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2360                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2361                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2363                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2364                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2365                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2366                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2367                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2368                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2369                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2371                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2372                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2373                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2374                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2376                           TILE_SPLIT(split_equal_to_row_size));
2377                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2378                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2379                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2380                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2381                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2382                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2383                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2384                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2385                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2386                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2387                           TILE_SPLIT(split_equal_to_row_size));
2388                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2389                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2390                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391                           PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2393                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2395                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2398                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2399                            PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2400                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2401                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2402                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2403                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2406                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2408                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2410                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2411                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2412                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2413                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2414                            PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2415                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2416                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2418                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2421                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2422                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2423                tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2424                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2425                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2426                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2427                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2428                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2429                            PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2430                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2431                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2432                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2433                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2434                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2435
2436                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2437                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2438                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2439                           NUM_BANKS(ADDR_SURF_16_BANK));
2440                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2442                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2443                           NUM_BANKS(ADDR_SURF_16_BANK));
2444                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2445                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2446                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2447                           NUM_BANKS(ADDR_SURF_16_BANK));
2448                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2450                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2451                           NUM_BANKS(ADDR_SURF_16_BANK));
2452                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2454                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2455                           NUM_BANKS(ADDR_SURF_8_BANK));
2456                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2458                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2459                           NUM_BANKS(ADDR_SURF_4_BANK));
2460                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2462                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2463                           NUM_BANKS(ADDR_SURF_2_BANK));
2464                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2466                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2467                           NUM_BANKS(ADDR_SURF_16_BANK));
2468                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2470                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2471                           NUM_BANKS(ADDR_SURF_16_BANK));
2472                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2475                            NUM_BANKS(ADDR_SURF_16_BANK));
2476                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2479                            NUM_BANKS(ADDR_SURF_8_BANK));
2480                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2482                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2483                            NUM_BANKS(ADDR_SURF_4_BANK));
2484                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487                            NUM_BANKS(ADDR_SURF_2_BANK));
2488                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491                            NUM_BANKS(ADDR_SURF_2_BANK));
2492
2493                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2494                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2495                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2496                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2497                break;
2498
2499        case 8:
2500                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2501                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2502                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2503                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2504                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2505                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2506                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2507                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2508                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2509                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2510                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2511                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2512                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2514                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2515                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2516                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2518                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2519                           TILE_SPLIT(split_equal_to_row_size));
2520                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2521                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2522                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2523                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2524                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2525                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2526                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2527                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2528                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2529                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2530                           TILE_SPLIT(split_equal_to_row_size));
2531                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2532                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2533                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2534                           PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2536                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2538                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2541                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2542                            PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2543                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2544                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2545                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2546                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2548                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2549                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2550                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2551                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2553                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2555                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2556                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2557                            PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2558                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2559                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2561                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2562                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2563                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2564                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2565                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2566                tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2567                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2568                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2570                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2571                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2572                            PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2573                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2574                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2575                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2576                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2577                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2578
2579                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2580                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2581                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2582                                NUM_BANKS(ADDR_SURF_16_BANK));
2583                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2585                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2586                                NUM_BANKS(ADDR_SURF_16_BANK));
2587                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2589                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2590                                NUM_BANKS(ADDR_SURF_16_BANK));
2591                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2593                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2594                                NUM_BANKS(ADDR_SURF_16_BANK));
2595                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2597                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2598                                NUM_BANKS(ADDR_SURF_8_BANK));
2599                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2600                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2601                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2602                                NUM_BANKS(ADDR_SURF_4_BANK));
2603                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2605                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2606                                NUM_BANKS(ADDR_SURF_2_BANK));
2607                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2609                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2610                                NUM_BANKS(ADDR_SURF_16_BANK));
2611                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2614                                NUM_BANKS(ADDR_SURF_16_BANK));
2615                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2617                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2618                                NUM_BANKS(ADDR_SURF_16_BANK));
2619                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2621                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2622                                NUM_BANKS(ADDR_SURF_16_BANK));
2623                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2626                                NUM_BANKS(ADDR_SURF_8_BANK));
2627                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2630                                NUM_BANKS(ADDR_SURF_4_BANK));
2631                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2634                                NUM_BANKS(ADDR_SURF_2_BANK));
2635
2636                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2637                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2638                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2639                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2640                break;
2641
2642        case 4:
2643                if (num_rbs == 4) {
2644                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2645                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2646                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2647                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2648                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2649                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2650                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2651                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2652                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2654                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2655                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2656                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2658                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2659                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2660                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2662                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2663                           TILE_SPLIT(split_equal_to_row_size));
2664                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2665                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2666                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2667                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2668                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2669                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2670                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2671                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2672                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2673                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2674                           TILE_SPLIT(split_equal_to_row_size));
2675                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2676                           PIPE_CONFIG(ADDR_SURF_P4_16x16));
2677                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2678                           PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2680                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2682                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2684                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2685                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2686                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2687                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2688                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2689                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2690                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2692                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2693                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2694                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2695                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2697                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2698                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2699                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2700                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2701                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2702                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2703                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2704                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2705                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2707                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2708                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2709                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2710                tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2711                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2712                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2713                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2714                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2715                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2716                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2717                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2718                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2719                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2720                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2721                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2722
2723                } else if (num_rbs < 4) {
2724                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2725                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2726                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2727                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2728                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2729                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2730                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2731                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2732                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2733                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2734                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2735                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2736                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2737                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2738                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2739                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2740                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2741                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2742                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2743                           TILE_SPLIT(split_equal_to_row_size));
2744                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2745                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2746                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2747                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2748                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2749                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2751                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2752                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2753                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2754                           TILE_SPLIT(split_equal_to_row_size));
2755                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2756                           PIPE_CONFIG(ADDR_SURF_P4_8x16));
2757                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2758                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2760                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2761                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2762                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2764                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2765                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2766                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2768                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2769                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2770                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2772                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2773                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2774                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2775                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2777                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2779                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2780                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2781                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2782                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2783                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2784                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2785                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2787                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2788                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2789                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2790                tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2792                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2793                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2794                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2795                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2796                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2797                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2798                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2799                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2800                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2801                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2802                }
2803
2804                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2805                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2806                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2807                                NUM_BANKS(ADDR_SURF_16_BANK));
2808                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2809                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2810                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2811                                NUM_BANKS(ADDR_SURF_16_BANK));
2812                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2813                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2814                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2815                                NUM_BANKS(ADDR_SURF_16_BANK));
2816                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2818                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2819                                NUM_BANKS(ADDR_SURF_16_BANK));
2820                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2822                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2823                                NUM_BANKS(ADDR_SURF_16_BANK));
2824                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2827                                NUM_BANKS(ADDR_SURF_8_BANK));
2828                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2829                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2830                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2831                                NUM_BANKS(ADDR_SURF_4_BANK));
2832                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2833                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2834                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2835                                NUM_BANKS(ADDR_SURF_16_BANK));
2836                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2837                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2838                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2839                                NUM_BANKS(ADDR_SURF_16_BANK));
2840                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2842                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2843                                NUM_BANKS(ADDR_SURF_16_BANK));
2844                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2846                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847                                NUM_BANKS(ADDR_SURF_16_BANK));
2848                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2849                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2850                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2851                                NUM_BANKS(ADDR_SURF_16_BANK));
2852                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2854                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2855                                NUM_BANKS(ADDR_SURF_8_BANK));
2856                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2859                                NUM_BANKS(ADDR_SURF_4_BANK));
2860
2861                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2862                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2863                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2864                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2865                break;
2866
2867        case 2:
2868                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2869                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2870                           PIPE_CONFIG(ADDR_SURF_P2) |
2871                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2872                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2873                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2874                           PIPE_CONFIG(ADDR_SURF_P2) |
2875                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2876                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2877                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2878                           PIPE_CONFIG(ADDR_SURF_P2) |
2879                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2880                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2881                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2882                           PIPE_CONFIG(ADDR_SURF_P2) |
2883                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2884                tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2886                           PIPE_CONFIG(ADDR_SURF_P2) |
2887                           TILE_SPLIT(split_equal_to_row_size));
2888                tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2889                           PIPE_CONFIG(ADDR_SURF_P2) |
2890                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2891                tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2892                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893                           PIPE_CONFIG(ADDR_SURF_P2) |
2894                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895                tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2896                           MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2897                           PIPE_CONFIG(ADDR_SURF_P2) |
2898                           TILE_SPLIT(split_equal_to_row_size));
2899                tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2900                           PIPE_CONFIG(ADDR_SURF_P2);
2901                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2902                           MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2903                           PIPE_CONFIG(ADDR_SURF_P2));
2904                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2905                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2906                            PIPE_CONFIG(ADDR_SURF_P2) |
2907                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2908                tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2909                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2910                            PIPE_CONFIG(ADDR_SURF_P2) |
2911                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2912                tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2913                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2914                            PIPE_CONFIG(ADDR_SURF_P2) |
2915                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2916                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2917                            PIPE_CONFIG(ADDR_SURF_P2) |
2918                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2919                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2920                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2921                            PIPE_CONFIG(ADDR_SURF_P2) |
2922                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2923                tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2924                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2925                            PIPE_CONFIG(ADDR_SURF_P2) |
2926                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2927                tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2928                            MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2929                            PIPE_CONFIG(ADDR_SURF_P2) |
2930                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2931                tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2932                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2933                            PIPE_CONFIG(ADDR_SURF_P2));
2934                tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2935                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2936                            PIPE_CONFIG(ADDR_SURF_P2) |
2937                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2938                tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2939                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2940                            PIPE_CONFIG(ADDR_SURF_P2) |
2941                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2942                tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2943                            MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2944                            PIPE_CONFIG(ADDR_SURF_P2) |
2945                            SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2946
2947                macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2948                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2949                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2950                                NUM_BANKS(ADDR_SURF_16_BANK));
2951                macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2952                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2953                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2954                                NUM_BANKS(ADDR_SURF_16_BANK));
2955                macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2956                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2957                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2958                                NUM_BANKS(ADDR_SURF_16_BANK));
2959                macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2960                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2961                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2962                                NUM_BANKS(ADDR_SURF_16_BANK));
2963                macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2964                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2965                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966                                NUM_BANKS(ADDR_SURF_16_BANK));
2967                macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2968                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2969                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2970                                NUM_BANKS(ADDR_SURF_16_BANK));
2971                macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2972                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2973                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2974                                NUM_BANKS(ADDR_SURF_8_BANK));
2975                macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2976                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2977                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2978                                NUM_BANKS(ADDR_SURF_16_BANK));
2979                macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2980                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2981                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2982                                NUM_BANKS(ADDR_SURF_16_BANK));
2983                macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2984                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2985                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2986                                NUM_BANKS(ADDR_SURF_16_BANK));
2987                macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2988                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2989                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990                                NUM_BANKS(ADDR_SURF_16_BANK));
2991                macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2993                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994                                NUM_BANKS(ADDR_SURF_16_BANK));
2995                macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2996                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2997                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998                                NUM_BANKS(ADDR_SURF_16_BANK));
2999                macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3000                                BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |

3001                                MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3002                                NUM_BANKS(ADDR_SURF_8_BANK));
3003
3004                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3005                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3006                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3007                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3008                break;
3009
3010        default:
3011                DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3012        }
3013}
3014
3015/**
3016 * cik_select_se_sh - select which SE, SH to address
3017 *
3018 * @rdev: radeon_device pointer
3019 * @se_num: shader engine to address
3020 * @sh_num: sh block to address
3021 *
3022 * Select which SE, SH combinations to address. Certain
3023 * registers are instanced per SE or SH.  0xffffffff means
3024 * broadcast to all SEs or SHs (CIK).
3025 */
3026static void cik_select_se_sh(struct radeon_device *rdev,
3027                             u32 se_num, u32 sh_num)
3028{
3029        u32 data = INSTANCE_BROADCAST_WRITES;
3030
3031        if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3032                data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3033        else if (se_num == 0xffffffff)
3034                data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3035        else if (sh_num == 0xffffffff)
3036                data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3037        else
3038                data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3039        WREG32(GRBM_GFX_INDEX, data);
3040}
3041
3042/**
3043 * cik_create_bitmask - create a bitmask
3044 *
3045 * @bit_width: length of the mask
3046 *
3047 * create a variable length bit mask (CIK).
3048 * Returns the bitmask.
3049 */
3050static u32 cik_create_bitmask(u32 bit_width)
3051{
3052        u32 i, mask = 0;
3053
3054        for (i = 0; i < bit_width; i++) {
3055                mask <<= 1;
3056                mask |= 1;
3057        }
3058        return mask;
3059}
3060
3061/**
3062 * cik_get_rb_disabled - computes the mask of disabled RBs
3063 *
3064 * @rdev: radeon_device pointer
3065 * @max_rb_num_per_se: max RBs (render backends) per SE (shader engine) for the asic
3066 * @sh_per_se: number of SH blocks per SE for the asic
3067 *
3068 * Calculates the bitmask of disabled RBs (CIK).
3069 * Returns the disabled RB bitmask.
3070 */
3071static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3072                              u32 max_rb_num_per_se,
3073                              u32 sh_per_se)
3074{
3075        u32 data, mask;
3076
3077        data = RREG32(CC_RB_BACKEND_DISABLE);
3078        if (data & 1)
3079                data &= BACKEND_DISABLE_MASK;
3080        else
3081                data = 0;
3082        data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3083
3084        data >>= BACKEND_DISABLE_SHIFT;
3085
3086        mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3087
3088        return data & mask;
3089}
3090
3091/**
3092 * cik_setup_rb - setup the RBs on the asic
3093 *
3094 * @rdev: radeon_device pointer
3095 * @se_num: number of SEs (shader engines) for the asic
3096 * @sh_per_se: number of SH blocks per SE for the asic
3097 * @max_rb_num_per_se: max RBs (render backends) per SE for the asic
3098 *
3099 * Configures per-SE/SH RB registers (CIK).
3100 */
3101static void cik_setup_rb(struct radeon_device *rdev,
3102                         u32 se_num, u32 sh_per_se,
3103                         u32 max_rb_num_per_se)
3104{
3105        int i, j;
3106        u32 data, mask;
3107        u32 disabled_rbs = 0;
3108        u32 enabled_rbs = 0;
3109
3110        for (i = 0; i < se_num; i++) {
3111                for (j = 0; j < sh_per_se; j++) {
3112                        cik_select_se_sh(rdev, i, j);
3113                        data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3114                        if (rdev->family == CHIP_HAWAII)
3115                                disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3116                        else
3117                                disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3118                }
3119        }
3120        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3121
3122        mask = 1;
3123        for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3124                if (!(disabled_rbs & mask))
3125                        enabled_rbs |= mask;
3126                mask <<= 1;
3127        }
3128
3129        rdev->config.cik.backend_enable_mask = enabled_rbs;
3130
3131        for (i = 0; i < se_num; i++) {
3132                cik_select_se_sh(rdev, i, 0xffffffff);
3133                data = 0;
3134                for (j = 0; j < sh_per_se; j++) {
3135                        switch (enabled_rbs & 3) {
3136                        case 0:
3137                                if (j == 0)
3138                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3139                                else
3140                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3141                                break;
3142                        case 1:
3143                                data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3144                                break;
3145                        case 2:
3146                                data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3147                                break;
3148                        case 3:
3149                        default:
3150                                data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3151                                break;
3152                        }
3153                        enabled_rbs >>= 2;
3154                }
3155                WREG32(PA_SC_RASTER_CONFIG, data);
3156        }
3157        cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3158}
3159
3160/**
3161 * cik_gpu_init - setup the 3D engine
3162 *
3163 * @rdev: radeon_device pointer
3164 *
3165 * Configures the 3D engine and tiling configuration
3166 * registers so that the 3D engine is usable.
3167 */
3168static void cik_gpu_init(struct radeon_device *rdev)
3169{
3170        u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3171        u32 mc_arb_ramcfg;
3172        u32 hdp_host_path_cntl;
3173        u32 tmp;
3174        int i, j;
3175
3176        switch (rdev->family) {
3177        case CHIP_BONAIRE:
3178                rdev->config.cik.max_shader_engines = 2;
3179                rdev->config.cik.max_tile_pipes = 4;
3180                rdev->config.cik.max_cu_per_sh = 7;
3181                rdev->config.cik.max_sh_per_se = 1;
3182                rdev->config.cik.max_backends_per_se = 2;
3183                rdev->config.cik.max_texture_channel_caches = 4;
3184                rdev->config.cik.max_gprs = 256;
3185                rdev->config.cik.max_gs_threads = 32;
3186                rdev->config.cik.max_hw_contexts = 8;
3187
3188                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3189                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3190                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3191                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3192                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3193                break;
3194        case CHIP_HAWAII:
3195                rdev->config.cik.max_shader_engines = 4;
3196                rdev->config.cik.max_tile_pipes = 16;
3197                rdev->config.cik.max_cu_per_sh = 11;
3198                rdev->config.cik.max_sh_per_se = 1;
3199                rdev->config.cik.max_backends_per_se = 4;
3200                rdev->config.cik.max_texture_channel_caches = 16;
3201                rdev->config.cik.max_gprs = 256;
3202                rdev->config.cik.max_gs_threads = 32;
3203                rdev->config.cik.max_hw_contexts = 8;
3204
3205                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3206                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3207                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3208                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3209                gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3210                break;
3211        case CHIP_KAVERI:
3212                rdev->config.cik.max_shader_engines = 1;
3213                rdev->config.cik.max_tile_pipes = 4;
3214                rdev->config.cik.max_cu_per_sh = 8;
3215                rdev->config.cik.max_backends_per_se = 2;
3216                rdev->config.cik.max_sh_per_se = 1;
3217                rdev->config.cik.max_texture_channel_caches = 4;
3218                rdev->config.cik.max_gprs = 256;
3219                rdev->config.cik.max_gs_threads = 16;
3220                rdev->config.cik.max_hw_contexts = 8;
3221
3222                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3223                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3224                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3225                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3226                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3227                break;
3228        case CHIP_KABINI:
3229        case CHIP_MULLINS:
3230        default:
3231                rdev->config.cik.max_shader_engines = 1;
3232                rdev->config.cik.max_tile_pipes = 2;
3233                rdev->config.cik.max_cu_per_sh = 2;
3234                rdev->config.cik.max_sh_per_se = 1;
3235                rdev->config.cik.max_backends_per_se = 1;
3236                rdev->config.cik.max_texture_channel_caches = 2;
3237                rdev->config.cik.max_gprs = 256;
3238                rdev->config.cik.max_gs_threads = 16;
3239                rdev->config.cik.max_hw_contexts = 8;
3240
3241                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3242                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3243                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3244                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3245                gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3246                break;
3247        }
3248
3249        /* Initialize HDP */
3250        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3251                WREG32((0x2c14 + j), 0x00000000);
3252                WREG32((0x2c18 + j), 0x00000000);
3253                WREG32((0x2c1c + j), 0x00000000);
3254                WREG32((0x2c20 + j), 0x00000000);
3255                WREG32((0x2c24 + j), 0x00000000);
3256        }
3257
3258        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3259        WREG32(SRBM_INT_CNTL, 0x1);
3260        WREG32(SRBM_INT_ACK, 0x1);
3261
3262        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3263
3264        RREG32(MC_SHARED_CHMAP);
3265        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3266
3267        rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3268        rdev->config.cik.mem_max_burst_length_bytes = 256;
3269        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3270        rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3271        if (rdev->config.cik.mem_row_size_in_kb > 4)
3272                rdev->config.cik.mem_row_size_in_kb = 4;
3273        /* XXX use MC settings? */
3274        rdev->config.cik.shader_engine_tile_size = 32;
3275        rdev->config.cik.num_gpus = 1;
3276        rdev->config.cik.multi_gpu_tile_size = 64;
3277
3278        /* fix up row size */
3279        gb_addr_config &= ~ROW_SIZE_MASK;
3280        switch (rdev->config.cik.mem_row_size_in_kb) {
3281        case 1:
3282        default:
3283                gb_addr_config |= ROW_SIZE(0);
3284                break;
3285        case 2:
3286                gb_addr_config |= ROW_SIZE(1);
3287                break;
3288        case 4:
3289                gb_addr_config |= ROW_SIZE(2);
3290                break;
3291        }
3292
3293        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3294         * not have bank info, so create a custom tiling dword.
3295         * bits 3:0   num_pipes
3296         * bits 7:4   num_banks
3297         * bits 11:8  group_size
3298         * bits 15:12 row_size
3299         */
3300        rdev->config.cik.tile_config = 0;
3301        switch (rdev->config.cik.num_tile_pipes) {
3302        case 1:
3303                rdev->config.cik.tile_config |= (0 << 0);
3304                break;
3305        case 2:
3306                rdev->config.cik.tile_config |= (1 << 0);
3307                break;
3308        case 4:
3309                rdev->config.cik.tile_config |= (2 << 0);
3310                break;
3311        case 8:
3312        default:
3313                /* XXX what about 12? */
3314                rdev->config.cik.tile_config |= (3 << 0);
3315                break;
3316        }
3317        rdev->config.cik.tile_config |=
3318                ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3319        rdev->config.cik.tile_config |=
3320                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3321        rdev->config.cik.tile_config |=
3322                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3323
3324        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3325        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3326        WREG32(DMIF_ADDR_CALC, gb_addr_config);
3327        WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3328        WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3329        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3330        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3331        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3332
3333        cik_tiling_mode_table_init(rdev);
3334
3335        cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3336                     rdev->config.cik.max_sh_per_se,
3337                     rdev->config.cik.max_backends_per_se);
3338
3339        rdev->config.cik.active_cus = 0;
3340        for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3341                for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3342                        rdev->config.cik.active_cus +=
3343                                hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3344                }
3345        }
3346
3347        /* set HW defaults for 3D engine */
3348        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3349
3350        WREG32(SX_DEBUG_1, 0x20);
3351
3352        WREG32(TA_CNTL_AUX, 0x00010000);
3353
3354        tmp = RREG32(SPI_CONFIG_CNTL);
3355        tmp |= 0x03000000;
3356        WREG32(SPI_CONFIG_CNTL, tmp);
3357
3358        WREG32(SQ_CONFIG, 1);
3359
3360        WREG32(DB_DEBUG, 0);
3361
3362        tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3363        tmp |= 0x00000400;
3364        WREG32(DB_DEBUG2, tmp);
3365
3366        tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3367        tmp |= 0x00020200;
3368        WREG32(DB_DEBUG3, tmp);
3369
3370        tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3371        tmp |= 0x00018208;
3372        WREG32(CB_HW_CONTROL, tmp);
3373
3374        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3375
3376        WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3377                                 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3378                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3379                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3380
3381        WREG32(VGT_NUM_INSTANCES, 1);
3382
3383        WREG32(CP_PERFMON_CNTL, 0);
3384
3385        WREG32(SQ_CONFIG, 0);
3386
3387        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3388                                          FORCE_EOV_MAX_REZ_CNT(255)));
3389
3390        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3391               AUTO_INVLD_EN(ES_AND_GS_AUTO));
3392
3393        WREG32(VGT_GS_VERTEX_REUSE, 16);
3394        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3395
3396        tmp = RREG32(HDP_MISC_CNTL);
3397        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3398        WREG32(HDP_MISC_CNTL, tmp);
3399
3400        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3401        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3402
3403        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3404        WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3405
3406        udelay(50);
3407}
3408
3409/*
3410 * GPU scratch registers helpers function.
3411 */
3412/**
3413 * cik_scratch_init - setup driver info for CP scratch regs
3414 *
3415 * @rdev: radeon_device pointer
3416 *
3417 * Set up the number and offset of the CP scratch registers.
3418 * NOTE: use of CP scratch registers is a legacy inferface and
3419 * is not used by default on newer asics (r6xx+).  On newer asics,
3420 * memory buffers are used for fences rather than scratch regs.
3421 */
3422static void cik_scratch_init(struct radeon_device *rdev)
3423{
3424        int i;
3425
3426        rdev->scratch.num_reg = 7;
3427        rdev->scratch.reg_base = SCRATCH_REG0;
3428        for (i = 0; i < rdev->scratch.num_reg; i++) {
3429                rdev->scratch.free[i] = true;
3430                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3431        }
3432}
3433
3434/**
3435 * cik_ring_test - basic gfx ring test
3436 *
3437 * @rdev: radeon_device pointer
3438 * @ring: radeon_ring structure holding ring information
3439 *
3440 * Allocate a scratch register and write to it using the gfx ring (CIK).
3441 * Provides a basic gfx ring test to verify that the ring is working.
3442 * Used by cik_cp_gfx_resume();
3443 * Returns 0 on success, error on failure.
3444 */
3445int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3446{
3447        uint32_t scratch;
3448        uint32_t tmp = 0;
3449        unsigned i;
3450        int r;
3451
3452        r = radeon_scratch_get(rdev, &scratch);
3453        if (r) {
3454                DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3455                return r;
3456        }
3457        WREG32(scratch, 0xCAFEDEAD);
3458        r = radeon_ring_lock(rdev, ring, 3);
3459        if (r) {
3460                DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3461                radeon_scratch_free(rdev, scratch);
3462                return r;
3463        }
3464        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3465        radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3466        radeon_ring_write(ring, 0xDEADBEEF);
3467        radeon_ring_unlock_commit(rdev, ring, false);
3468
3469        for (i = 0; i < rdev->usec_timeout; i++) {
3470                tmp = RREG32(scratch);
3471                if (tmp == 0xDEADBEEF)
3472                        break;
3473                udelay(1);
3474        }
3475        if (i < rdev->usec_timeout) {
3476                DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3477        } else {
3478                DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3479                          ring->idx, scratch, tmp);
3480                r = -EINVAL;
3481        }
3482        radeon_scratch_free(rdev, scratch);
3483        return r;
3484}
3485
3486/**
3487 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3488 *
3489 * @rdev: radeon_device pointer
3490 * @ridx: radeon ring index
3491 *
3492 * Emits an hdp flush on the cp.
3493 */
3494static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3495                                       int ridx)
3496{
3497        struct radeon_ring *ring = &rdev->ring[ridx];
3498        u32 ref_and_mask;
3499
3500        switch (ring->idx) {
3501        case CAYMAN_RING_TYPE_CP1_INDEX:
3502        case CAYMAN_RING_TYPE_CP2_INDEX:
3503        default:
3504                switch (ring->me) {
3505                case 0:
3506                        ref_and_mask = CP2 << ring->pipe;
3507                        break;
3508                case 1:
3509                        ref_and_mask = CP6 << ring->pipe;
3510                        break;
3511                default:
3512                        return;
3513                }
3514                break;
3515        case RADEON_RING_TYPE_GFX_INDEX:
3516                ref_and_mask = CP0;
3517                break;
3518        }
3519
3520        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3521        radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3522                                 WAIT_REG_MEM_FUNCTION(3) |  /* == */
3523                                 WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3524        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3525        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3526        radeon_ring_write(ring, ref_and_mask);
3527        radeon_ring_write(ring, ref_and_mask);
3528        radeon_ring_write(ring, 0x20); /* poll interval */
3529}
3530
3531/**
3532 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3533 *
3534 * @rdev: radeon_device pointer
3535 * @fence: radeon fence object
3536 *
3537 * Emits a fence sequnce number on the gfx ring and flushes
3538 * GPU caches.
3539 */
3540void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3541                             struct radeon_fence *fence)
3542{
3543        struct radeon_ring *ring = &rdev->ring[fence->ring];
3544        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3545
3546        /* Workaround for cache flush problems. First send a dummy EOP
3547         * event down the pipe with seq one below.
3548         */
3549        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3550        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3551                                 EOP_TC_ACTION_EN |
3552                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3553                                 EVENT_INDEX(5)));
3554        radeon_ring_write(ring, addr & 0xfffffffc);
3555        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3556                                DATA_SEL(1) | INT_SEL(0));
3557        radeon_ring_write(ring, fence->seq - 1);
3558        radeon_ring_write(ring, 0);
3559
3560        /* Then send the real EOP event down the pipe. */
3561        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3562        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3563                                 EOP_TC_ACTION_EN |
3564                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3565                                 EVENT_INDEX(5)));
3566        radeon_ring_write(ring, addr & 0xfffffffc);
3567        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3568        radeon_ring_write(ring, fence->seq);
3569        radeon_ring_write(ring, 0);
3570}
3571
3572/**
3573 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3574 *
3575 * @rdev: radeon_device pointer
3576 * @fence: radeon fence object
3577 *
3578 * Emits a fence sequnce number on the compute ring and flushes
3579 * GPU caches.
3580 */
3581void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3582                                 struct radeon_fence *fence)
3583{
3584        struct radeon_ring *ring = &rdev->ring[fence->ring];
3585        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3586
3587        /* RELEASE_MEM - flush caches, send int */
3588        radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3589        radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3590                                 EOP_TC_ACTION_EN |
3591                                 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3592                                 EVENT_INDEX(5)));
3593        radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3594        radeon_ring_write(ring, addr & 0xfffffffc);
3595        radeon_ring_write(ring, upper_32_bits(addr));
3596        radeon_ring_write(ring, fence->seq);
3597        radeon_ring_write(ring, 0);
3598}
3599
3600/**
3601 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3602 *
3603 * @rdev: radeon_device pointer
3604 * @ring: radeon ring buffer object
3605 * @semaphore: radeon semaphore object
3606 * @emit_wait: Is this a sempahore wait?
3607 *
3608 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3609 * from running ahead of semaphore waits.
3610 */
3611bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3612                             struct radeon_ring *ring,
3613                             struct radeon_semaphore *semaphore,
3614                             bool emit_wait)
3615{
3616        uint64_t addr = semaphore->gpu_addr;
3617        unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3618
3619        radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3620        radeon_ring_write(ring, lower_32_bits(addr));
3621        radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3622
3623        if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3624                /* Prevent the PFP from running ahead of the semaphore wait */
3625                radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3626                radeon_ring_write(ring, 0x0);
3627        }
3628
3629        return true;
3630}
3631
3632/**
3633 * cik_copy_cpdma - copy pages using the CP DMA engine
3634 *
3635 * @rdev: radeon_device pointer
3636 * @src_offset: src GPU address
3637 * @dst_offset: dst GPU address
3638 * @num_gpu_pages: number of GPU pages to xfer
3639 * @resv: reservation object to sync to
3640 *
3641 * Copy GPU paging using the CP DMA engine (CIK+).
3642 * Used by the radeon ttm implementation to move pages if
3643 * registered as the asic copy callback.
3644 */
3645struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3646                                    uint64_t src_offset, uint64_t dst_offset,
3647                                    unsigned num_gpu_pages,
3648                                    struct dma_resv *resv)
3649{
3650        struct radeon_fence *fence;
3651        struct radeon_sync sync;
3652        int ring_index = rdev->asic->copy.blit_ring_index;
3653        struct radeon_ring *ring = &rdev->ring[ring_index];
3654        u32 size_in_bytes, cur_size_in_bytes, control;
3655        int i, num_loops;
3656        int r = 0;
3657
3658        radeon_sync_create(&sync);
3659
3660        size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3661        num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3662        r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3663        if (r) {
3664                DRM_ERROR("radeon: moving bo (%d).\n", r);
3665                radeon_sync_free(rdev, &sync, NULL);
3666                return ERR_PTR(r);
3667        }
3668
3669        radeon_sync_resv(rdev, &sync, resv, false);
3670        radeon_sync_rings(rdev, &sync, ring->idx);
3671
3672        for (i = 0; i < num_loops; i++) {
3673                cur_size_in_bytes = size_in_bytes;
3674                if (cur_size_in_bytes > 0x1fffff)
3675                        cur_size_in_bytes = 0x1fffff;
3676                size_in_bytes -= cur_size_in_bytes;
3677                control = 0;
3678                if (size_in_bytes == 0)
3679                        control |= PACKET3_DMA_DATA_CP_SYNC;
3680                radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3681                radeon_ring_write(ring, control);
3682                radeon_ring_write(ring, lower_32_bits(src_offset));
3683                radeon_ring_write(ring, upper_32_bits(src_offset));
3684                radeon_ring_write(ring, lower_32_bits(dst_offset));
3685                radeon_ring_write(ring, upper_32_bits(dst_offset));
3686                radeon_ring_write(ring, cur_size_in_bytes);
3687                src_offset += cur_size_in_bytes;
3688                dst_offset += cur_size_in_bytes;
3689        }
3690
3691        r = radeon_fence_emit(rdev, &fence, ring->idx);
3692        if (r) {
3693                radeon_ring_unlock_undo(rdev, ring);
3694                radeon_sync_free(rdev, &sync, NULL);
3695                return ERR_PTR(r);
3696        }
3697
3698        radeon_ring_unlock_commit(rdev, ring, false);
3699        radeon_sync_free(rdev, &sync, fence);
3700
3701        return fence;
3702}
3703
3704/*
3705 * IB stuff
3706 */
3707/**
3708 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3709 *
3710 * @rdev: radeon_device pointer
3711 * @ib: radeon indirect buffer object
3712 *
3713 * Emits a DE (drawing engine) or CE (constant engine) IB
3714 * on the gfx ring.  IBs are usually generated by userspace
3715 * acceleration drivers and submitted to the kernel for
3716 * scheduling on the ring.  This function schedules the IB
3717 * on the gfx ring for execution by the GPU.
3718 */
3719void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3720{
3721        struct radeon_ring *ring = &rdev->ring[ib->ring];
3722        unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3723        u32 header, control = INDIRECT_BUFFER_VALID;
3724
3725        if (ib->is_const_ib) {
3726                /* set switch buffer packet before const IB */
3727                radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3728                radeon_ring_write(ring, 0);
3729
3730                header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3731        } else {
3732                u32 next_rptr;
3733                if (ring->rptr_save_reg) {
3734                        next_rptr = ring->wptr + 3 + 4;
3735                        radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3736                        radeon_ring_write(ring, ((ring->rptr_save_reg -
3737                                                  PACKET3_SET_UCONFIG_REG_START) >> 2));
3738                        radeon_ring_write(ring, next_rptr);
3739                } else if (rdev->wb.enabled) {
3740                        next_rptr = ring->wptr + 5 + 4;
3741                        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3742                        radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3743                        radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3744                        radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3745                        radeon_ring_write(ring, next_rptr);
3746                }
3747
3748                header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3749        }
3750
3751        control |= ib->length_dw | (vm_id << 24);
3752
3753        radeon_ring_write(ring, header);
3754        radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3755        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3756        radeon_ring_write(ring, control);
3757}
3758
3759/**
3760 * cik_ib_test - basic gfx ring IB test
3761 *
3762 * @rdev: radeon_device pointer
3763 * @ring: radeon_ring structure holding ring information
3764 *
3765 * Allocate an IB and execute it on the gfx ring (CIK).
3766 * Provides a basic gfx ring test to verify that IBs are working.
3767 * Returns 0 on success, error on failure.
3768 */
3769int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3770{
3771        struct radeon_ib ib;
3772        uint32_t scratch;
3773        uint32_t tmp = 0;
3774        unsigned i;
3775        int r;
3776
3777        r = radeon_scratch_get(rdev, &scratch);
3778        if (r) {
3779                DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3780                return r;
3781        }
3782        WREG32(scratch, 0xCAFEDEAD);
3783        r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3784        if (r) {
3785                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3786                radeon_scratch_free(rdev, scratch);
3787                return r;
3788        }
3789        ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3790        ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3791        ib.ptr[2] = 0xDEADBEEF;
3792        ib.length_dw = 3;
3793        r = radeon_ib_schedule(rdev, &ib, NULL, false);
3794        if (r) {
3795                radeon_scratch_free(rdev, scratch);
3796                radeon_ib_free(rdev, &ib);
3797                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3798                return r;
3799        }
3800        r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3801                RADEON_USEC_IB_TEST_TIMEOUT));
3802        if (r < 0) {
3803                DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3804                radeon_scratch_free(rdev, scratch);
3805                radeon_ib_free(rdev, &ib);
3806                return r;
3807        } else if (r == 0) {
3808                DRM_ERROR("radeon: fence wait timed out.\n");
3809                radeon_scratch_free(rdev, scratch);
3810                radeon_ib_free(rdev, &ib);
3811                return -ETIMEDOUT;
3812        }
3813        r = 0;
3814        for (i = 0; i < rdev->usec_timeout; i++) {
3815                tmp = RREG32(scratch);
3816                if (tmp == 0xDEADBEEF)
3817                        break;
3818                udelay(1);
3819        }
3820        if (i < rdev->usec_timeout) {
3821                DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3822        } else {
3823                DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3824                          scratch, tmp);
3825                r = -EINVAL;
3826        }
3827        radeon_scratch_free(rdev, scratch);
3828        radeon_ib_free(rdev, &ib);
3829        return r;
3830}
3831
3832/*
3833 * CP.
3834 * On CIK, gfx and compute now have independant command processors.
3835 *
3836 * GFX
3837 * Gfx consists of a single ring and can process both gfx jobs and
3838 * compute jobs.  The gfx CP consists of three microengines (ME):
3839 * PFP - Pre-Fetch Parser
3840 * ME - Micro Engine
3841 * CE - Constant Engine
3842 * The PFP and ME make up what is considered the Drawing Engine (DE).
3843 * The CE is an asynchronous engine used for updating buffer desciptors
3844 * used by the DE so that they can be loaded into cache in parallel
3845 * while the DE is processing state update packets.
3846 *
3847 * Compute
3848 * The compute CP consists of two microengines (ME):
3849 * MEC1 - Compute MicroEngine 1
3850 * MEC2 - Compute MicroEngine 2
3851 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3852 * The queues are exposed to userspace and are programmed directly
3853 * by the compute runtime.
3854 */
3855/**
3856 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3857 *
3858 * @rdev: radeon_device pointer
3859 * @enable: enable or disable the MEs
3860 *
3861 * Halts or unhalts the gfx MEs.
3862 */
3863static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3864{
3865        if (enable)
3866                WREG32(CP_ME_CNTL, 0);
3867        else {
3868                if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3869                        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3870                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3871                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3872        }
3873        udelay(50);
3874}
3875
3876/**
3877 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3878 *
3879 * @rdev: radeon_device pointer
3880 *
3881 * Loads the gfx PFP, ME, and CE ucode.
3882 * Returns 0 for success, -EINVAL if the ucode is not available.
3883 */
3884static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3885{
3886        int i;
3887
3888        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3889                return -EINVAL;
3890
3891        cik_cp_gfx_enable(rdev, false);
3892
3893        if (rdev->new_fw) {
3894                const struct gfx_firmware_header_v1_0 *pfp_hdr =
3895                        (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3896                const struct gfx_firmware_header_v1_0 *ce_hdr =
3897                        (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3898                const struct gfx_firmware_header_v1_0 *me_hdr =
3899                        (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3900                const __le32 *fw_data;
3901                u32 fw_size;
3902
3903                radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3904                radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3905                radeon_ucode_print_gfx_hdr(&me_hdr->header);
3906
3907                /* PFP */
3908                fw_data = (const __le32 *)
3909                        (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3910                fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3911                WREG32(CP_PFP_UCODE_ADDR, 0);
3912                for (i = 0; i < fw_size; i++)
3913                        WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3914                WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3915
3916                /* CE */
3917                fw_data = (const __le32 *)
3918                        (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3919                fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3920                WREG32(CP_CE_UCODE_ADDR, 0);
3921                for (i = 0; i < fw_size; i++)
3922                        WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3923                WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3924
3925                /* ME */
3926                fw_data = (const __be32 *)
3927                        (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3928                fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3929                WREG32(CP_ME_RAM_WADDR, 0);
3930                for (i = 0; i < fw_size; i++)
3931                        WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3932                WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3933                WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
3934        } else {
3935                const __be32 *fw_data;
3936
3937                /* PFP */
3938                fw_data = (const __be32 *)rdev->pfp_fw->data;
3939                WREG32(CP_PFP_UCODE_ADDR, 0);
3940                for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3941                        WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3942                WREG32(CP_PFP_UCODE_ADDR, 0);
3943
3944                /* CE */
3945                fw_data = (const __be32 *)rdev->ce_fw->data;
3946                WREG32(CP_CE_UCODE_ADDR, 0);
3947                for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3948                        WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3949                WREG32(CP_CE_UCODE_ADDR, 0);
3950
3951                /* ME */
3952                fw_data = (const __be32 *)rdev->me_fw->data;
3953                WREG32(CP_ME_RAM_WADDR, 0);
3954                for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3955                        WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3956                WREG32(CP_ME_RAM_WADDR, 0);
3957        }
3958
3959        return 0;
3960}
3961
3962/**
3963 * cik_cp_gfx_start - start the gfx ring
3964 *
3965 * @rdev: radeon_device pointer
3966 *
3967 * Enables the ring and loads the clear state context and other
3968 * packets required to init the ring.
3969 * Returns 0 for success, error for failure.
3970 */
3971static int cik_cp_gfx_start(struct radeon_device *rdev)
3972{
3973        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3974        int r, i;
3975
3976        /* init the CP */
3977        WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3978        WREG32(CP_ENDIAN_SWAP, 0);
3979        WREG32(CP_DEVICE_ID, 1);
3980
3981        cik_cp_gfx_enable(rdev, true);
3982
3983        r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3984        if (r) {
3985                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3986                return r;
3987        }
3988
3989        /* init the CE partitions.  CE only used for gfx on CIK */
3990        radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3991        radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3992        radeon_ring_write(ring, 0x8000);
3993        radeon_ring_write(ring, 0x8000);
3994
3995        /* setup clear context state */
3996        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3997        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3998
3999        radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4000        radeon_ring_write(ring, 0x80000000);

4001        radeon_ring_write(ring, 0x80000000);
4002
4003        for (i = 0; i < cik_default_size; i++)
4004                radeon_ring_write(ring, cik_default_state[i]);
4005
4006        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4007        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4008
4009        /* set clear context state */
4010        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4011        radeon_ring_write(ring, 0);
4012
4013        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4014        radeon_ring_write(ring, 0x00000316);
4015        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4016        radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4017
4018        radeon_ring_unlock_commit(rdev, ring, false);
4019
4020        return 0;
4021}
4022
4023/**
4024 * cik_cp_gfx_fini - stop the gfx ring
4025 *
4026 * @rdev: radeon_device pointer
4027 *
4028 * Stop the gfx ring and tear down the driver ring
4029 * info.
4030 */
4031static void cik_cp_gfx_fini(struct radeon_device *rdev)
4032{
4033        cik_cp_gfx_enable(rdev, false);
4034        radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4035}
4036
4037/**
4038 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4039 *
4040 * @rdev: radeon_device pointer
4041 *
4042 * Program the location and size of the gfx ring buffer
4043 * and test it to make sure it's working.
4044 * Returns 0 for success, error for failure.
4045 */
4046static int cik_cp_gfx_resume(struct radeon_device *rdev)
4047{
4048        struct radeon_ring *ring;
4049        u32 tmp;
4050        u32 rb_bufsz;
4051        u64 rb_addr;
4052        int r;
4053
4054        WREG32(CP_SEM_WAIT_TIMER, 0x0);
4055        if (rdev->family != CHIP_HAWAII)
4056                WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4057
4058        /* Set the write pointer delay */
4059        WREG32(CP_RB_WPTR_DELAY, 0);
4060
4061        /* set the RB to use vmid 0 */
4062        WREG32(CP_RB_VMID, 0);
4063
4064        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4065
4066        /* ring 0 - compute and gfx */
4067        /* Set ring buffer size */
4068        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4069        rb_bufsz = order_base_2(ring->ring_size / 8);
4070        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4071#ifdef __BIG_ENDIAN
4072        tmp |= BUF_SWAP_32BIT;
4073#endif
4074        WREG32(CP_RB0_CNTL, tmp);
4075
4076        /* Initialize the ring buffer's read and write pointers */
4077        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4078        ring->wptr = 0;
4079        WREG32(CP_RB0_WPTR, ring->wptr);
4080
4081        /* set the wb address wether it's enabled or not */
4082        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4083        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4084
4085        /* scratch register shadowing is no longer supported */
4086        WREG32(SCRATCH_UMSK, 0);
4087
4088        if (!rdev->wb.enabled)
4089                tmp |= RB_NO_UPDATE;
4090
4091        mdelay(1);
4092        WREG32(CP_RB0_CNTL, tmp);
4093
4094        rb_addr = ring->gpu_addr >> 8;
4095        WREG32(CP_RB0_BASE, rb_addr);
4096        WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4097
4098        /* start the ring */
4099        cik_cp_gfx_start(rdev);
4100        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4101        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4102        if (r) {
4103                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4104                return r;
4105        }
4106
4107        if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4108                radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4109
4110        return 0;
4111}
4112
4113u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4114                     struct radeon_ring *ring)
4115{
4116        u32 rptr;
4117
4118        if (rdev->wb.enabled)
4119                rptr = rdev->wb.wb[ring->rptr_offs/4];
4120        else
4121                rptr = RREG32(CP_RB0_RPTR);
4122
4123        return rptr;
4124}
4125
4126u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4127                     struct radeon_ring *ring)
4128{
4129        return RREG32(CP_RB0_WPTR);
4130}
4131
4132void cik_gfx_set_wptr(struct radeon_device *rdev,
4133                      struct radeon_ring *ring)
4134{
4135        WREG32(CP_RB0_WPTR, ring->wptr);
4136        (void)RREG32(CP_RB0_WPTR);
4137}
4138
4139u32 cik_compute_get_rptr(struct radeon_device *rdev,
4140                         struct radeon_ring *ring)
4141{
4142        u32 rptr;
4143
4144        if (rdev->wb.enabled) {
4145                rptr = rdev->wb.wb[ring->rptr_offs/4];
4146        } else {
4147                mutex_lock(&rdev->srbm_mutex);
4148                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4149                rptr = RREG32(CP_HQD_PQ_RPTR);
4150                cik_srbm_select(rdev, 0, 0, 0, 0);
4151                mutex_unlock(&rdev->srbm_mutex);
4152        }
4153
4154        return rptr;
4155}
4156
4157u32 cik_compute_get_wptr(struct radeon_device *rdev,
4158                         struct radeon_ring *ring)
4159{
4160        u32 wptr;
4161
4162        if (rdev->wb.enabled) {
4163                /* XXX check if swapping is necessary on BE */
4164                wptr = rdev->wb.wb[ring->wptr_offs/4];
4165        } else {
4166                mutex_lock(&rdev->srbm_mutex);
4167                cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4168                wptr = RREG32(CP_HQD_PQ_WPTR);
4169                cik_srbm_select(rdev, 0, 0, 0, 0);
4170                mutex_unlock(&rdev->srbm_mutex);
4171        }
4172
4173        return wptr;
4174}
4175
4176void cik_compute_set_wptr(struct radeon_device *rdev,
4177                          struct radeon_ring *ring)
4178{
4179        /* XXX check if swapping is necessary on BE */
4180        rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4181        WDOORBELL32(ring->doorbell_index, ring->wptr);
4182}
4183
4184static void cik_compute_stop(struct radeon_device *rdev,
4185                             struct radeon_ring *ring)
4186{
4187        u32 j, tmp;
4188
4189        cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4190        /* Disable wptr polling. */
4191        tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4192        tmp &= ~WPTR_POLL_EN;
4193        WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4194        /* Disable HQD. */
4195        if (RREG32(CP_HQD_ACTIVE) & 1) {
4196                WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4197                for (j = 0; j < rdev->usec_timeout; j++) {
4198                        if (!(RREG32(CP_HQD_ACTIVE) & 1))
4199                                break;
4200                        udelay(1);
4201                }
4202                WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4203                WREG32(CP_HQD_PQ_RPTR, 0);
4204                WREG32(CP_HQD_PQ_WPTR, 0);
4205        }
4206        cik_srbm_select(rdev, 0, 0, 0, 0);
4207}
4208
4209/**
4210 * cik_cp_compute_enable - enable/disable the compute CP MEs
4211 *
4212 * @rdev: radeon_device pointer
4213 * @enable: enable or disable the MEs
4214 *
4215 * Halts or unhalts the compute MEs.
4216 */
4217static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4218{
4219        if (enable)
4220                WREG32(CP_MEC_CNTL, 0);
4221        else {
4222                /*
4223                 * To make hibernation reliable we need to clear compute ring
4224                 * configuration before halting the compute ring.
4225                 */
4226                mutex_lock(&rdev->srbm_mutex);
4227                cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4228                cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4229                mutex_unlock(&rdev->srbm_mutex);
4230
4231                WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4232                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4233                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4234        }
4235        udelay(50);
4236}
4237
4238/**
4239 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4240 *
4241 * @rdev: radeon_device pointer
4242 *
4243 * Loads the compute MEC1&2 ucode.
4244 * Returns 0 for success, -EINVAL if the ucode is not available.
4245 */
4246static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4247{
4248        int i;
4249
4250        if (!rdev->mec_fw)
4251                return -EINVAL;
4252
4253        cik_cp_compute_enable(rdev, false);
4254
4255        if (rdev->new_fw) {
4256                const struct gfx_firmware_header_v1_0 *mec_hdr =
4257                        (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4258                const __le32 *fw_data;
4259                u32 fw_size;
4260
4261                radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4262
4263                /* MEC1 */
4264                fw_data = (const __le32 *)
4265                        (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4266                fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4267                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4268                for (i = 0; i < fw_size; i++)
4269                        WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4270                WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4271
4272                /* MEC2 */
4273                if (rdev->family == CHIP_KAVERI) {
4274                        const struct gfx_firmware_header_v1_0 *mec2_hdr =
4275                                (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4276
4277                        fw_data = (const __le32 *)
4278                                (rdev->mec2_fw->data +
4279                                 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4280                        fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4281                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4282                        for (i = 0; i < fw_size; i++)
4283                                WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4284                        WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4285                }
4286        } else {
4287                const __be32 *fw_data;
4288
4289                /* MEC1 */
4290                fw_data = (const __be32 *)rdev->mec_fw->data;
4291                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4292                for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4293                        WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4294                WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4295
4296                if (rdev->family == CHIP_KAVERI) {
4297                        /* MEC2 */
4298                        fw_data = (const __be32 *)rdev->mec_fw->data;
4299                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4300                        for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4301                                WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4302                        WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4303                }
4304        }
4305
4306        return 0;
4307}
4308
4309/**
4310 * cik_cp_compute_start - start the compute queues
4311 *
4312 * @rdev: radeon_device pointer
4313 *
4314 * Enable the compute queues.
4315 * Returns 0 for success, error for failure.
4316 */
4317static int cik_cp_compute_start(struct radeon_device *rdev)
4318{
4319        cik_cp_compute_enable(rdev, true);
4320
4321        return 0;
4322}
4323
43