LXR linux/drivers/gpu/drm/amd/amdgpu/amdgpu

   1/*
   2 * Copyright 2013 Advanced Micro Devices, Inc.
   3 * All Rights Reserved.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the
   7 * "Software"), to deal in the Software without restriction, including
   8 * without limitation the rights to use, copy, modify, merge, publish,
   9 * distribute, sub license, and/or sell copies of the Software, and to
  10 * permit persons to whom the Software is furnished to do so, subject to
  11 * the following conditions:
  12 *
  13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20 *
  21 * The above copyright notice and this permission notice (including the
  22 * next paragraph) shall be included in all copies or substantial portions
  23 * of the Software.
  24 *
  25 * Authors: Christian König <christian.koenig@amd.com>
  26 */
  27
  28#include <linux/firmware.h>
  29#include <linux/module.h>
  30
  31#include <drm/drm.h>
  32#include <drm/drm_drv.h>
  33
  34#include "amdgpu.h"
  35#include "amdgpu_pm.h"
  36#include "amdgpu_vce.h"
  37#include "cikd.h"
  38
  39/* 1 second timeout */
  40#define VCE_IDLE_TIMEOUT        msecs_to_jiffies(1000)
  41
  42/* Firmware Names */
  43#ifdef CONFIG_DRM_AMDGPU_CIK
  44#define FIRMWARE_BONAIRE        "amdgpu/bonaire_vce.bin"
  45#define FIRMWARE_KABINI "amdgpu/kabini_vce.bin"
  46#define FIRMWARE_KAVERI "amdgpu/kaveri_vce.bin"
  47#define FIRMWARE_HAWAII "amdgpu/hawaii_vce.bin"
  48#define FIRMWARE_MULLINS        "amdgpu/mullins_vce.bin"
  49#endif
  50#define FIRMWARE_TONGA          "amdgpu/tonga_vce.bin"
  51#define FIRMWARE_CARRIZO        "amdgpu/carrizo_vce.bin"
  52#define FIRMWARE_FIJI           "amdgpu/fiji_vce.bin"
  53#define FIRMWARE_STONEY         "amdgpu/stoney_vce.bin"
  54#define FIRMWARE_POLARIS10      "amdgpu/polaris10_vce.bin"
  55#define FIRMWARE_POLARIS11      "amdgpu/polaris11_vce.bin"
  56#define FIRMWARE_POLARIS12      "amdgpu/polaris12_vce.bin"
  57#define FIRMWARE_VEGAM          "amdgpu/vegam_vce.bin"
  58
  59#define FIRMWARE_VEGA10         "amdgpu/vega10_vce.bin"
  60#define FIRMWARE_VEGA12         "amdgpu/vega12_vce.bin"
  61#define FIRMWARE_VEGA20         "amdgpu/vega20_vce.bin"
  62
  63#ifdef CONFIG_DRM_AMDGPU_CIK
  64MODULE_FIRMWARE(FIRMWARE_BONAIRE);
  65MODULE_FIRMWARE(FIRMWARE_KABINI);
  66MODULE_FIRMWARE(FIRMWARE_KAVERI);
  67MODULE_FIRMWARE(FIRMWARE_HAWAII);
  68MODULE_FIRMWARE(FIRMWARE_MULLINS);
  69#endif
  70MODULE_FIRMWARE(FIRMWARE_TONGA);
  71MODULE_FIRMWARE(FIRMWARE_CARRIZO);
  72MODULE_FIRMWARE(FIRMWARE_FIJI);
  73MODULE_FIRMWARE(FIRMWARE_STONEY);
  74MODULE_FIRMWARE(FIRMWARE_POLARIS10);
  75MODULE_FIRMWARE(FIRMWARE_POLARIS11);
  76MODULE_FIRMWARE(FIRMWARE_POLARIS12);
  77MODULE_FIRMWARE(FIRMWARE_VEGAM);
  78
  79MODULE_FIRMWARE(FIRMWARE_VEGA10);
  80MODULE_FIRMWARE(FIRMWARE_VEGA12);
  81MODULE_FIRMWARE(FIRMWARE_VEGA20);
  82
  83static void amdgpu_vce_idle_work_handler(struct work_struct *work);
  84static int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
  85                                     struct amdgpu_bo *bo,
  86                                     struct dma_fence **fence);
  87static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
  88                                      bool direct, struct dma_fence **fence);
  89
  90/**
  91 * amdgpu_vce_sw_init - allocate memory, load vce firmware
  92 *
  93 * @adev: amdgpu_device pointer
  94 * @size: size for the new BO
  95 *
  96 * First step to get VCE online, allocate memory and load the firmware
  97 */
  98int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
  99{
 100        const char *fw_name;
 101        const struct common_firmware_header *hdr;
 102        unsigned ucode_version, version_major, version_minor, binary_id;
 103        int i, r;
 104
 105        switch (adev->asic_type) {
 106#ifdef CONFIG_DRM_AMDGPU_CIK
 107        case CHIP_BONAIRE:
 108                fw_name = FIRMWARE_BONAIRE;
 109                break;
 110        case CHIP_KAVERI:
 111                fw_name = FIRMWARE_KAVERI;
 112                break;
 113        case CHIP_KABINI:
 114                fw_name = FIRMWARE_KABINI;
 115                break;
 116        case CHIP_HAWAII:
 117                fw_name = FIRMWARE_HAWAII;
 118                break;
 119        case CHIP_MULLINS:
 120                fw_name = FIRMWARE_MULLINS;
 121                break;
 122#endif
 123        case CHIP_TONGA:
 124                fw_name = FIRMWARE_TONGA;
 125                break;
 126        case CHIP_CARRIZO:
 127                fw_name = FIRMWARE_CARRIZO;
 128                break;
 129        case CHIP_FIJI:
 130                fw_name = FIRMWARE_FIJI;
 131                break;
 132        case CHIP_STONEY:
 133                fw_name = FIRMWARE_STONEY;
 134                break;
 135        case CHIP_POLARIS10:
 136                fw_name = FIRMWARE_POLARIS10;
 137                break;
 138        case CHIP_POLARIS11:
 139                fw_name = FIRMWARE_POLARIS11;
 140                break;
 141        case CHIP_POLARIS12:
 142                fw_name = FIRMWARE_POLARIS12;
 143                break;
 144        case CHIP_VEGAM:
 145                fw_name = FIRMWARE_VEGAM;
 146                break;
 147        case CHIP_VEGA10:
 148                fw_name = FIRMWARE_VEGA10;
 149                break;
 150        case CHIP_VEGA12:
 151                fw_name = FIRMWARE_VEGA12;
 152                break;
 153        case CHIP_VEGA20:
 154                fw_name = FIRMWARE_VEGA20;
 155                break;
 156
 157        default:
 158                return -EINVAL;
 159        }
 160
 161        r = request_firmware(&adev->vce.fw, fw_name, adev->dev);
 162        if (r) {
 163                dev_err(adev->dev, "amdgpu_vce: Can't load firmware \"%s\"\n",
 164                        fw_name);
 165                return r;
 166        }
 167
 168        r = amdgpu_ucode_validate(adev->vce.fw);
 169        if (r) {
 170                dev_err(adev->dev, "amdgpu_vce: Can't validate firmware \"%s\"\n",
 171                        fw_name);
 172                release_firmware(adev->vce.fw);
 173                adev->vce.fw = NULL;
 174                return r;
 175        }
 176
 177        hdr = (const struct common_firmware_header *)adev->vce.fw->data;
 178
 179        ucode_version = le32_to_cpu(hdr->ucode_version);
 180        version_major = (ucode_version >> 20) & 0xfff;
 181        version_minor = (ucode_version >> 8) & 0xfff;
 182        binary_id = ucode_version & 0xff;
 183        DRM_INFO("Found VCE firmware Version: %d.%d Binary ID: %d\n",
 184                version_major, version_minor, binary_id);
 185        adev->vce.fw_version = ((version_major << 24) | (version_minor << 16) |
 186                                (binary_id << 8));
 187
 188        r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
 189                                    AMDGPU_GEM_DOMAIN_VRAM, &adev->vce.vcpu_bo,
 190                                    &adev->vce.gpu_addr, &adev->vce.cpu_addr);
 191        if (r) {
 192                dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
 193                return r;
 194        }
 195
 196        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
 197                atomic_set(&adev->vce.handles[i], 0);
 198                adev->vce.filp[i] = NULL;
 199        }
 200
 201        INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler);
 202        mutex_init(&adev->vce.idle_mutex);
 203
 204        return 0;
 205}
 206
 207/**
 208 * amdgpu_vce_sw_fini - free memory
 209 *
 210 * @adev: amdgpu_device pointer
 211 *
 212 * Last step on VCE teardown, free firmware memory
 213 */
 214int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
 215{
 216        unsigned i;
 217
 218        if (adev->vce.vcpu_bo == NULL)
 219                return 0;
 220
 221        cancel_delayed_work_sync(&adev->vce.idle_work);
 222        drm_sched_entity_destroy(&adev->vce.entity);
 223
 224        amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr,
 225                (void **)&adev->vce.cpu_addr);
 226
 227        for (i = 0; i < adev->vce.num_rings; i++)
 228                amdgpu_ring_fini(&adev->vce.ring[i]);
 229
 230        release_firmware(adev->vce.fw);
 231        mutex_destroy(&adev->vce.idle_mutex);
 232
 233        return 0;
 234}
 235
 236/**
 237 * amdgpu_vce_entity_init - init entity
 238 *
 239 * @adev: amdgpu_device pointer
 240 *
 241 */
 242int amdgpu_vce_entity_init(struct amdgpu_device *adev)
 243{
 244        struct amdgpu_ring *ring;
 245        struct drm_gpu_scheduler *sched;
 246        int r;
 247
 248        ring = &adev->vce.ring[0];
 249        sched = &ring->sched;
 250        r = drm_sched_entity_init(&adev->vce.entity, DRM_SCHED_PRIORITY_NORMAL,
 251                                  &sched, 1, NULL);
 252        if (r != 0) {
 253                DRM_ERROR("Failed setting up VCE run queue.\n");
 254                return r;
 255        }
 256
 257        return 0;
 258}
 259
 260/**
 261 * amdgpu_vce_suspend - unpin VCE fw memory
 262 *
 263 * @adev: amdgpu_device pointer
 264 *
 265 */
 266int amdgpu_vce_suspend(struct amdgpu_device *adev)
 267{
 268        int i;
 269
 270        cancel_delayed_work_sync(&adev->vce.idle_work);
 271
 272        if (adev->vce.vcpu_bo == NULL)
 273                return 0;
 274
 275        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
 276                if (atomic_read(&adev->vce.handles[i]))
 277                        break;
 278
 279        if (i == AMDGPU_MAX_VCE_HANDLES)
 280                return 0;
 281
 282        /* TODO: suspending running encoding sessions isn't supported */
 283        return -EINVAL;
 284}
 285
 286/**
 287 * amdgpu_vce_resume - pin VCE fw memory
 288 *
 289 * @adev: amdgpu_device pointer
 290 *
 291 */
 292int amdgpu_vce_resume(struct amdgpu_device *adev)
 293{
 294        void *cpu_addr;
 295        const struct common_firmware_header *hdr;
 296        unsigned offset;
 297        int r, idx;
 298
 299        if (adev->vce.vcpu_bo == NULL)
 300                return -EINVAL;
 301
 302        r = amdgpu_bo_reserve(adev->vce.vcpu_bo, false);
 303        if (r) {
 304                dev_err(adev->dev, "(%d) failed to reserve VCE bo\n", r);
 305                return r;
 306        }
 307
 308        r = amdgpu_bo_kmap(adev->vce.vcpu_bo, &cpu_addr);
 309        if (r) {
 310                amdgpu_bo_unreserve(adev->vce.vcpu_bo);
 311                dev_err(adev->dev, "(%d) VCE map failed\n", r);
 312                return r;
 313        }
 314
 315        hdr = (const struct common_firmware_header *)adev->vce.fw->data;
 316        offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
 317
 318        if (drm_dev_enter(&adev->ddev, &idx)) {
 319                memcpy_toio(cpu_addr, adev->vce.fw->data + offset,
 320                            adev->vce.fw->size - offset);
 321                drm_dev_exit(idx);
 322        }
 323
 324        amdgpu_bo_kunmap(adev->vce.vcpu_bo);
 325
 326        amdgpu_bo_unreserve(adev->vce.vcpu_bo);
 327
 328        return 0;
 329}
 330
 331/**
 332 * amdgpu_vce_idle_work_handler - power off VCE
 333 *
 334 * @work: pointer to work structure
 335 *
 336 * power of VCE when it's not used any more
 337 */
 338static void amdgpu_vce_idle_work_handler(struct work_struct *work)
 339{
 340        struct amdgpu_device *adev =
 341                container_of(work, struct amdgpu_device, vce.idle_work.work);
 342        unsigned i, count = 0;
 343
 344        for (i = 0; i < adev->vce.num_rings; i++)
 345                count += amdgpu_fence_count_emitted(&adev->vce.ring[i]);
 346
 347        if (count == 0) {
 348                if (adev->pm.dpm_enabled) {
 349                        amdgpu_dpm_enable_vce(adev, false);
 350                } else {
 351                        amdgpu_asic_set_vce_clocks(adev, 0, 0);
 352                        amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 353                                                               AMD_PG_STATE_GATE);
 354                        amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 355                                                               AMD_CG_STATE_GATE);
 356                }
 357        } else {
 358                schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT);
 359        }
 360}
 361
 362/**
 363 * amdgpu_vce_ring_begin_use - power up VCE
 364 *
 365 * @ring: amdgpu ring
 366 *
 367 * Make sure VCE is powerd up when we want to use it
 368 */
 369void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring)
 370{
 371        struct amdgpu_device *adev = ring->adev;
 372        bool set_clocks;
 373
 374        if (amdgpu_sriov_vf(adev))
 375                return;
 376
 377        mutex_lock(&adev->vce.idle_mutex);
 378        set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work);
 379        if (set_clocks) {
 380                if (adev->pm.dpm_enabled) {
 381                        amdgpu_dpm_enable_vce(adev, true);
 382                } else {
 383                        amdgpu_asic_set_vce_clocks(adev, 53300, 40000);
 384                        amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 385                                                               AMD_CG_STATE_UNGATE);
 386                        amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 387                                                               AMD_PG_STATE_UNGATE);
 388
 389                }
 390        }
 391        mutex_unlock(&adev->vce.idle_mutex);
 392}
 393
 394/**
 395 * amdgpu_vce_ring_end_use - power VCE down
 396 *
 397 * @ring: amdgpu ring
 398 *
 399 * Schedule work to power VCE down again
 400 */
 401void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring)
 402{
 403        if (!amdgpu_sriov_vf(ring->adev))
 404                schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT);
 405}
 406
 407/**
 408 * amdgpu_vce_free_handles - free still open VCE handles
 409 *
 410 * @adev: amdgpu_device pointer
 411 * @filp: drm file pointer
 412 *
 413 * Close all VCE handles still open by this file pointer
 414 */
 415void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 416{
 417        struct amdgpu_ring *ring = &adev->vce.ring[0];
 418        int i, r;
 419        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
 420                uint32_t handle = atomic_read(&adev->vce.handles[i]);
 421
 422                if (!handle || adev->vce.filp[i] != filp)
 423                        continue;
 424
 425                r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL);
 426                if (r)
 427                        DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
 428
 429                adev->vce.filp[i] = NULL;
 430                atomic_set(&adev->vce.handles[i], 0);
 431        }
 432}
 433
 434/**
 435 * amdgpu_vce_get_create_msg - generate a VCE create msg
 436 *
 437 * @ring: ring we should submit the msg to
 438 * @handle: VCE session handle to use
 439 * @bo: amdgpu object for which we query the offset
 440 * @fence: optional fence to return
 441 *
 442 * Open up a stream for HW test
 443 */
 444static int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 445                                     struct amdgpu_bo *bo,
 446                                     struct dma_fence **fence)
 447{
 448        const unsigned ib_size_dw = 1024;
 449        struct amdgpu_job *job;
 450        struct amdgpu_ib *ib;
 451        struct dma_fence *f = NULL;
 452        uint64_t addr;
 453        int i, r;
 454
 455        r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4,
 456                                     AMDGPU_IB_POOL_DIRECT, &job);
 457        if (r)
 458                return r;
 459
 460        ib = &job->ibs[0];
 461
 462        addr = amdgpu_bo_gpu_offset(bo);
 463
 464        /* stitch together an VCE create msg */
 465        ib->length_dw = 0;
 466        ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
 467        ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
 468        ib->ptr[ib->length_dw++] = handle;
 469
 470        if ((ring->adev->vce.fw_version >> 24) >= 52)
 471                ib->ptr[ib->length_dw++] = 0x00000040; /* len */
 472        else
 473                ib->ptr[ib->length_dw++] = 0x00000030; /* len */
 474        ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */
 475        ib->ptr[ib->length_dw++] = 0x00000000;
 476        ib->ptr[ib->length_dw++] = 0x00000042;
 477        ib->ptr[ib->length_dw++] = 0x0000000a;
 478        ib->ptr[ib->length_dw++] = 0x00000001;
 479        ib->ptr[ib->length_dw++] = 0x00000080;
 480        ib->ptr[ib->length_dw++] = 0x00000060;
 481        ib->ptr[ib->length_dw++] = 0x00000100;
 482        ib->ptr[ib->length_dw++] = 0x00000100;
 483        ib->ptr[ib->length_dw++] = 0x0000000c;
 484        ib->ptr[ib->length_dw++] = 0x00000000;
 485        if ((ring->adev->vce.fw_version >> 24) >= 52) {
 486                ib->ptr[ib->length_dw++] = 0x00000000;
 487                ib->ptr[ib->length_dw++] = 0x00000000;
 488                ib->ptr[ib->length_dw++] = 0x00000000;
 489                ib->ptr[ib->length_dw++] = 0x00000000;
 490        }
 491
 492        ib->ptr[ib->length_dw++] = 0x00000014; /* len */
 493        ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
 494        ib->ptr[ib->length_dw++] = upper_32_bits(addr);
 495        ib->ptr[ib->length_dw++] = addr;
 496        ib->ptr[ib->length_dw++] = 0x00000001;
 497
 498        for (i = ib->length_dw; i < ib_size_dw; ++i)
 499                ib->ptr[i] = 0x0;
 500
 501        r = amdgpu_job_submit_direct(job, ring, &f);
 502        if (r)
 503                goto err;
 504
 505        if (fence)
 506                *fence = dma_fence_get(f);
 507        dma_fence_put(f);
 508        return 0;
 509
 510err:
 511        amdgpu_job_free(job);
 512        return r;
 513}
 514
 515/**
 516 * amdgpu_vce_get_destroy_msg - generate a VCE destroy msg
 517 *
 518 * @ring: ring we should submit the msg to
 519 * @handle: VCE session handle to use
 520 * @direct: direct or delayed pool
 521 * @fence: optional fence to return
 522 *
 523 * Close up a stream for HW test or if userspace failed to do so
 524 */
 525static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 526                                      bool direct, struct dma_fence **fence)
 527{
 528        const unsigned ib_size_dw = 1024;
 529        struct amdgpu_job *job;
 530        struct amdgpu_ib *ib;
 531        struct dma_fence *f = NULL;
 532        int i, r;
 533
 534        r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4,
 535                                     direct ? AMDGPU_IB_POOL_DIRECT :
 536                                     AMDGPU_IB_POOL_DELAYED, &job);
 537        if (r)
 538                return r;
 539
 540        ib = &job->ibs[0];
 541
 542        /* stitch together an VCE destroy msg */
 543        ib->length_dw = 0;
 544        ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
 545        ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
 546        ib->ptr[ib->length_dw++] = handle;
 547
 548        ib->ptr[ib->length_dw++] = 0x00000020; /* len */
 549        ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
 550        ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */
 551        ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */
 552        ib->ptr[ib->length_dw++] = 0x00000000;
 553        ib->ptr[ib->length_dw++] = 0x00000000;
 554        ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */
 555        ib->ptr[ib->length_dw++] = 0x00000000;
 556
 557        ib->ptr[ib->length_dw++] = 0x00000008; /* len */
 558        ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */
 559
 560        for (i = ib->length_dw; i < ib_size_dw; ++i)
 561                ib->ptr[i] = 0x0;
 562
 563        if (direct)
 564                r = amdgpu_job_submit_direct(job, ring, &f);
 565        else
 566                r = amdgpu_job_submit(job, &ring->adev->vce.entity,
 567                                      AMDGPU_FENCE_OWNER_UNDEFINED, &f);
 568        if (r)
 569                goto err;
 570
 571        if (fence)
 572                *fence = dma_fence_get(f);
 573        dma_fence_put(f);
 574        return 0;
 575
 576err:
 577        amdgpu_job_free(job);
 578        return r;
 579}
 580
 581/**
 582 * amdgpu_vce_validate_bo - make sure not to cross 4GB boundary
 583 *
 584 * @p: parser context
 585 * @ib_idx: indirect buffer to use
 586 * @lo: address of lower dword
 587 * @hi: address of higher dword
 588 * @size: minimum size
 589 * @index: bs/fb index
 590 *
 591 * Make sure that no BO cross a 4GB boundary.
 592 */
 593static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser *p, uint32_t ib_idx,
 594                                  int lo, int hi, unsigned size, int32_t index)
 595{
 596        int64_t offset = ((uint64_t)size) * ((int64_t)index);
 597        struct ttm_operation_ctx ctx = { false, false };
 598        struct amdgpu_bo_va_mapping *mapping;
 599        unsigned i, fpfn, lpfn;
 600        struct amdgpu_bo *bo;
 601        uint64_t addr;
 602        int r;
 603
 604        addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
 605               ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
 606        if (index >= 0) {
 607                addr += offset;
 608                fpfn = PAGE_ALIGN(offset) >> PAGE_SHIFT;
 609                lpfn = 0x100000000ULL >> PAGE_SHIFT;
 610        } else {
 611                fpfn = 0;
 612                lpfn = (0x100000000ULL - PAGE_ALIGN(offset)) >> PAGE_SHIFT;
 613        }
 614
 615        r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
 616        if (r) {
 617                DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
 618                          addr, lo, hi, size, index);
 619                return r;
 620        }
 621
 622        for (i = 0; i < bo->placement.num_placement; ++i) {
 623                bo->placements[i].fpfn = max(bo->placements[i].fpfn, fpfn);
 624                bo->placements[i].lpfn = bo->placements[i].lpfn ?
 625                        min(bo->placements[i].lpfn, lpfn) : lpfn;
 626        }
 627        return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 628}
 629
 630
 631/**
 632 * amdgpu_vce_cs_reloc - command submission relocation
 633 *
 634 * @p: parser context
 635 * @ib_idx: indirect buffer to use
 636 * @lo: address of lower dword
 637 * @hi: address of higher dword
 638 * @size: minimum size
 639 * @index: bs/fb index
 640 *
 641 * Patch relocation inside command stream with real buffer address
 642 */
 643static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
 644                               int lo, int hi, unsigned size, uint32_t index)
 645{
 646        struct amdgpu_bo_va_mapping *mapping;
 647        struct amdgpu_bo *bo;
 648        uint64_t addr;
 649        int r;
 650
 651        if (index == 0xffffffff)
 652                index = 0;
 653
 654        addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
 655               ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
 656        addr += ((uint64_t)size) * ((uint64_t)index);
 657
 658        r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
 659        if (r) {
 660                DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
 661                          addr, lo, hi, size, index);
 662                return r;
 663        }
 664
 665        if ((addr + (uint64_t)size) >
 666            (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
 667                DRM_ERROR("BO too small for addr 0x%010Lx %d %d\n",
 668                          addr, lo, hi);
 669                return -EINVAL;
 670        }
 671
 672        addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE;
 673        addr += amdgpu_bo_gpu_offset(bo);
 674        addr -= ((uint64_t)size) * ((uint64_t)index);
 675
 676        amdgpu_set_ib_value(p, ib_idx, lo, lower_32_bits(addr));
 677        amdgpu_set_ib_value(p, ib_idx, hi, upper_32_bits(addr));
 678
 679        return 0;
 680}
 681
 682/**
 683 * amdgpu_vce_validate_handle - validate stream handle
 684 *
 685 * @p: parser context
 686 * @handle: handle to validate
 687 * @allocated: allocated a new handle?
 688 *
 689 * Validates the handle and return the found session index or -EINVAL
 690 * we we don't have another free session index.
 691 */
 692static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p,
 693                                      uint32_t handle, uint32_t *allocated)
 694{
 695        unsigned i;
 696
 697        /* validate the handle */
 698        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
 699                if (atomic_read(&p->adev->vce.handles[i]) == handle) {
 700                        if (p->adev->vce.filp[i] != p->filp) {
 701                                DRM_ERROR("VCE handle collision detected!\n");
 702                                return -EINVAL;
 703                        }
 704                        return i;
 705                }
 706        }
 707
 708        /* handle not found try to alloc a new one */
 709        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
 710                if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) {
 711                        p->adev->vce.filp[i] = p->filp;
 712                        p->adev->vce.img_size[i] = 0;
 713                        *allocated |= 1 << i;
 714                        return i;
 715                }
 716        }
 717
 718        DRM_ERROR("No more free VCE handles!\n");
 719        return -EINVAL;
 720}
 721
 722/**
 723 * amdgpu_vce_ring_parse_cs - parse and validate the command stream
 724 *
 725 * @p: parser context
 726 * @ib_idx: indirect buffer to use
 727 */
 728int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 729{
 730        struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
 731        unsigned fb_idx = 0, bs_idx = 0;
 732        int session_idx = -1;
 733        uint32_t destroyed = 0;
 734        uint32_t created = 0;
 735        uint32_t allocated = 0;
 736        uint32_t tmp, handle = 0;
 737        uint32_t *size = &tmp;
 738        unsigned idx;
 739        int i, r = 0;
 740
 741        p->job->vm = NULL;
 742        ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
 743
 744        for (idx = 0; idx < ib->length_dw;) {
 745                uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
 746                uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
 747
 748                if ((len < 8) || (len & 3)) {
 749                        DRM_ERROR("invalid VCE command length (%d)!\n", len);
 750                        r = -EINVAL;
 751                        goto out;
 752                }
 753
 754                switch (cmd) {
 755                case 0x00000002: /* task info */
 756                        fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
 757                        bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
 758                        break;
 759
 760                case 0x03000001: /* encode */
 761                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 10,
 762                                                   idx + 9, 0, 0);
 763                        if (r)
 764                                goto out;
 765
 766                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 12,
 767                                                   idx + 11, 0, 0);
 768                        if (r)
 769                                goto out;
 770                        break;
 771
 772                case 0x05000001: /* context buffer */
 773                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
 774                                                   idx + 2, 0, 0);
 775                        if (r)
 776                                goto out;
 777                        break;
 778
 779                case 0x05000004: /* video bitstream buffer */
 780                        tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
 781                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
 782                                                   tmp, bs_idx);
 783                        if (r)
 784                                goto out;
 785                        break;
 786
 787                case 0x05000005: /* feedback buffer */
 788                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
 789                                                   4096, fb_idx);
 790                        if (r)
 791                                goto out;
 792                        break;
 793
 794                case 0x0500000d: /* MV buffer */
 795                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
 796                                                        idx + 2, 0, 0);
 797                        if (r)
 798                                goto out;
 799
 800                        r = amdgpu_vce_validate_bo(p, ib_idx, idx + 8,
 801                                                        idx + 7, 0, 0);
 802                        if (r)
 803                                goto out;
 804                        break;
 805                }
 806
 807                idx += len / 4;
 808        }
 809
 810        for (idx = 0; idx < ib->length_dw;) {
 811                uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
 812                uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
 813
 814                switch (cmd) {
 815                case 0x00000001: /* session */
 816                        handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
 817                        session_idx = amdgpu_vce_validate_handle(p, handle,
 818                                                                 &allocated);
 819                        if (session_idx < 0) {
 820                                r = session_idx;
 821                                goto out;
 822                        }
 823                        size = &p->adev->vce.img_size[session_idx];
 824                        break;
 825
 826                case 0x00000002: /* task info */
 827                        fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
 828                        bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
 829                        break;
 830
 831                case 0x01000001: /* create */
 832                        created |= 1 << session_idx;
 833                        if (destroyed & (1 << session_idx)) {
 834                                destroyed &= ~(1 << session_idx);
 835                                allocated |= 1 << session_idx;
 836
 837                        } else if (!(allocated & (1 << session_idx))) {
 838                                DRM_ERROR("Handle already in use!\n");
 839                                r = -EINVAL;
 840                                goto out;
 841                        }
 842
 843                        *size = amdgpu_get_ib_value(p, ib_idx, idx + 8) *
 844                                amdgpu_get_ib_value(p, ib_idx, idx + 10) *
 845                                8 * 3 / 2;
 846                        break;
 847
 848                case 0x04000001: /* config extension */
 849                case 0x04000002: /* pic control */
 850                case 0x04000005: /* rate control */
 851                case 0x04000007: /* motion estimation */
 852                case 0x04000008: /* rdo */
 853                case 0x04000009: /* vui */
 854                case 0x05000002: /* auxiliary buffer */
 855                case 0x05000009: /* clock table */
 856                        break;
 857
 858                case 0x0500000c: /* hw config */
 859                        switch (p->adev->asic_type) {
 860#ifdef CONFIG_DRM_AMDGPU_CIK
 861                        case CHIP_KAVERI:
 862                        case CHIP_MULLINS:
 863#endif
 864                        case CHIP_CARRIZO:
 865                                break;
 866                        default:
 867                                r = -EINVAL;
 868                                goto out;
 869                        }
 870                        break;
 871
 872                case 0x03000001: /* encode */
 873                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9,
 874                                                *size, 0);
 875                        if (r)
 876                                goto out;
 877
 878                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 12, idx + 11,
 879                                                *size / 3, 0);
 880                        if (r)
 881                                goto out;
 882                        break;
 883
 884                case 0x02000001: /* destroy */
 885                        destroyed |= 1 << session_idx;
 886                        break;
 887
 888                case 0x05000001: /* context buffer */
 889                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
 890                                                *size * 2, 0);
 891                        if (r)
 892                                goto out;
 893                        break;
 894
 895                case 0x05000004: /* video bitstream buffer */
 896                        tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
 897                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
 898                                                tmp, bs_idx);
 899                        if (r)
 900                                goto out;
 901                        break;
 902
 903                case 0x05000005: /* feedback buffer */
 904                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
 905                                                4096, fb_idx);
 906                        if (r)
 907                                goto out;
 908                        break;
 909
 910                case 0x0500000d: /* MV buffer */
 911                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3,
 912                                                        idx + 2, *size, 0);
 913                        if (r)
 914                                goto out;
 915
 916                        r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 8,
 917                                                        idx + 7, *size / 12, 0);
 918                        if (r)
 919                                goto out;
 920                        break;
 921
 922                default:
 923                        DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
 924                        r = -EINVAL;
 925                        goto out;
 926                }
 927
 928                if (session_idx == -1) {
 929                        DRM_ERROR("no session command at start of IB\n");
 930                        r = -EINVAL;
 931                        goto out;
 932                }
 933
 934                idx += len / 4;
 935        }
 936
 937        if (allocated & ~created) {
 938                DRM_ERROR("New session without create command!\n");
 939                r = -ENOENT;
 940        }
 941
 942out:
 943        if (!r) {
 944                /* No error, free all destroyed handle slots */
 945                tmp = destroyed;
 946        } else {
 947                /* Error during parsing, free all allocated handle slots */
 948                tmp = allocated;
 949        }
 950
 951        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
 952                if (tmp & (1 << i))
 953                        atomic_set(&p->adev->vce.handles[i], 0);
 954
 955        return r;
 956}
 957
 958/**
 959 * amdgpu_vce_ring_parse_cs_vm - parse the command stream in VM mode
 960 *
 961 * @p: parser context
 962 * @ib_idx: indirect buffer to use
 963 */
 964int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 965{
 966        struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
 967        int session_idx = -1;
 968        uint32_t destroyed = 0;
 969        uint32_t created = 0;
 970        uint32_t allocated = 0;
 971        uint32_t tmp, handle = 0;
 972        int i, r = 0, idx = 0;
 973
 974        while (idx < ib->length_dw) {
 975                uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
 976                uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
 977
 978                if ((len < 8) || (len & 3)) {
 979                        DRM_ERROR("invalid VCE command length (%d)!\n", len);
 980                        r = -EINVAL;
 981                        goto out;
 982                }
 983
 984                switch (cmd) {
 985                case 0x00000001: /* session */
 986                        handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
 987                        session_idx = amdgpu_vce_validate_handle(p, handle,
 988                                                                 &allocated);
 989                        if (session_idx < 0) {
 990                                r = session_idx;
 991                                goto out;
 992                        }
 993                        break;
 994
 995                case 0x01000001: /* create */
 996                        created |= 1 << session_idx;
 997                        if (destroyed & (1 << session_idx)) {
 998                                destroyed &= ~(1 << session_idx);
 999                                allocated |= 1 << session_idx;
1000

1001                        } else if (!(allocated & (1 << session_idx))) {
1002                                DRM_ERROR("Handle already in use!\n");
1003                                r = -EINVAL;
1004                                goto out;
1005                        }
1006
1007                        break;
1008
1009                case 0x02000001: /* destroy */
1010                        destroyed |= 1 << session_idx;
1011                        break;
1012
1013                default:
1014                        break;
1015                }
1016
1017                if (session_idx == -1) {
1018                        DRM_ERROR("no session command at start of IB\n");
1019                        r = -EINVAL;
1020                        goto out;
1021                }
1022
1023                idx += len / 4;
1024        }
1025
1026        if (allocated & ~created) {
1027                DRM_ERROR("New session without create command!\n");
1028                r = -ENOENT;
1029        }
1030
1031out:
1032        if (!r) {
1033                /* No error, free all destroyed handle slots */
1034                tmp = destroyed;
1035                amdgpu_ib_free(p->adev, ib, NULL);
1036        } else {
1037                /* Error during parsing, free all allocated handle slots */
1038                tmp = allocated;
1039        }
1040
1041        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
1042                if (tmp & (1 << i))
1043                        atomic_set(&p->adev->vce.handles[i], 0);
1044
1045        return r;
1046}
1047
1048/**
1049 * amdgpu_vce_ring_emit_ib - execute indirect buffer
1050 *
1051 * @ring: engine to use
1052 * @job: job to retrieve vmid from
1053 * @ib: the IB to execute
1054 * @flags: unused
1055 *
1056 */
1057void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring,
1058                                struct amdgpu_job *job,
1059                                struct amdgpu_ib *ib,
1060                                uint32_t flags)
1061{
1062        amdgpu_ring_write(ring, VCE_CMD_IB);
1063        amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1064        amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1065        amdgpu_ring_write(ring, ib->length_dw);
1066}
1067
1068/**
1069 * amdgpu_vce_ring_emit_fence - add a fence command to the ring
1070 *
1071 * @ring: engine to use
1072 * @addr: address
1073 * @seq: sequence number
1074 * @flags: fence related flags
1075 *
1076 */
1077void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1078                                unsigned flags)
1079{
1080        WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1081
1082        amdgpu_ring_write(ring, VCE_CMD_FENCE);
1083        amdgpu_ring_write(ring, addr);
1084        amdgpu_ring_write(ring, upper_32_bits(addr));
1085        amdgpu_ring_write(ring, seq);
1086        amdgpu_ring_write(ring, VCE_CMD_TRAP);
1087        amdgpu_ring_write(ring, VCE_CMD_END);
1088}
1089
1090/**
1091 * amdgpu_vce_ring_test_ring - test if VCE ring is working
1092 *
1093 * @ring: the engine to test on
1094 *
1095 */
1096int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
1097{
1098        struct amdgpu_device *adev = ring->adev;
1099        uint32_t rptr;
1100        unsigned i;
1101        int r, timeout = adev->usec_timeout;
1102
1103        /* skip ring test for sriov*/
1104        if (amdgpu_sriov_vf(adev))
1105                return 0;
1106
1107        r = amdgpu_ring_alloc(ring, 16);
1108        if (r)
1109                return r;
1110
1111        rptr = amdgpu_ring_get_rptr(ring);
1112
1113        amdgpu_ring_write(ring, VCE_CMD_END);
1114        amdgpu_ring_commit(ring);
1115
1116        for (i = 0; i < timeout; i++) {
1117                if (amdgpu_ring_get_rptr(ring) != rptr)
1118                        break;
1119                udelay(1);
1120        }
1121
1122        if (i >= timeout)
1123                r = -ETIMEDOUT;
1124
1125        return r;
1126}
1127
1128/**
1129 * amdgpu_vce_ring_test_ib - test if VCE IBs are working
1130 *
1131 * @ring: the engine to test on
1132 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1133 *
1134 */
1135int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1136{
1137        struct dma_fence *fence = NULL;
1138        struct amdgpu_bo *bo = NULL;
1139        long r;
1140
1141        /* skip vce ring1/2 ib test for now, since it's not reliable */
1142        if (ring != &ring->adev->vce.ring[0])
1143                return 0;
1144
1145        r = amdgpu_bo_create_reserved(ring->adev, 512, PAGE_SIZE,
1146                                      AMDGPU_GEM_DOMAIN_VRAM,
1147                                      &bo, NULL, NULL);
1148        if (r)
1149                return r;
1150
1151        r = amdgpu_vce_get_create_msg(ring, 1, bo, NULL);
1152        if (r)
1153                goto error;
1154
1155        r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence);
1156        if (r)
1157                goto error;
1158
1159        r = dma_fence_wait_timeout(fence, false, timeout);
1160        if (r == 0)
1161                r = -ETIMEDOUT;
1162        else if (r > 0)
1163                r = 0;
1164
1165error:
1166        dma_fence_put(fence);
1167        amdgpu_bo_unreserve(bo);
1168        amdgpu_bo_free_kernel(&bo, NULL, NULL);
1169        return r;
1170}
1171