linux/drivers/gpu/drm/i915/gt/selftest_lrc.c
<<
>>
Prefs
   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2018 Intel Corporation
   4 */
   5
   6#include <linux/prime_numbers.h>
   7
   8#include "i915_selftest.h"
   9#include "intel_engine_heartbeat.h"
  10#include "intel_engine_pm.h"
  11#include "intel_reset.h"
  12#include "intel_ring.h"
  13#include "selftest_engine_heartbeat.h"
  14#include "selftests/i915_random.h"
  15#include "selftests/igt_flush_test.h"
  16#include "selftests/igt_live_test.h"
  17#include "selftests/igt_spinner.h"
  18#include "selftests/lib_sw_fence.h"
  19#include "shmem_utils.h"
  20
  21#include "gem/selftests/igt_gem_utils.h"
  22#include "gem/selftests/mock_context.h"
  23
  24#define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
  25#define NUM_GPR 16
  26#define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
  27
  28static struct i915_vma *create_scratch(struct intel_gt *gt)
  29{
  30        return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
  31}
  32
  33static bool is_active(struct i915_request *rq)
  34{
  35        if (i915_request_is_active(rq))
  36                return true;
  37
  38        if (i915_request_on_hold(rq))
  39                return true;
  40
  41        if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
  42                return true;
  43
  44        return false;
  45}
  46
  47static int wait_for_submit(struct intel_engine_cs *engine,
  48                           struct i915_request *rq,
  49                           unsigned long timeout)
  50{
  51        /* Ignore our own attempts to suppress excess tasklets */
  52        tasklet_hi_schedule(&engine->execlists.tasklet);
  53
  54        timeout += jiffies;
  55        do {
  56                bool done = time_after(jiffies, timeout);
  57
  58                if (i915_request_completed(rq)) /* that was quick! */
  59                        return 0;
  60
  61                /* Wait until the HW has acknowleged the submission (or err) */
  62                intel_engine_flush_submission(engine);
  63                if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
  64                        return 0;
  65
  66                if (done)
  67                        return -ETIME;
  68
  69                cond_resched();
  70        } while (1);
  71}
  72
  73static int emit_semaphore_signal(struct intel_context *ce, void *slot)
  74{
  75        const u32 offset =
  76                i915_ggtt_offset(ce->engine->status_page.vma) +
  77                offset_in_page(slot);
  78        struct i915_request *rq;
  79        u32 *cs;
  80
  81        rq = intel_context_create_request(ce);
  82        if (IS_ERR(rq))
  83                return PTR_ERR(rq);
  84
  85        cs = intel_ring_begin(rq, 4);
  86        if (IS_ERR(cs)) {
  87                i915_request_add(rq);
  88                return PTR_ERR(cs);
  89        }
  90
  91        *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
  92        *cs++ = offset;
  93        *cs++ = 0;
  94        *cs++ = 1;
  95
  96        intel_ring_advance(rq, cs);
  97
  98        rq->sched.attr.priority = I915_PRIORITY_BARRIER;
  99        i915_request_add(rq);
 100        return 0;
 101}
 102
 103static int context_flush(struct intel_context *ce, long timeout)
 104{
 105        struct i915_request *rq;
 106        struct dma_fence *fence;
 107        int err = 0;
 108
 109        rq = intel_engine_create_kernel_request(ce->engine);
 110        if (IS_ERR(rq))
 111                return PTR_ERR(rq);
 112
 113        fence = i915_active_fence_get(&ce->timeline->last_request);
 114        if (fence) {
 115                i915_request_await_dma_fence(rq, fence);
 116                dma_fence_put(fence);
 117        }
 118
 119        rq = i915_request_get(rq);
 120        i915_request_add(rq);
 121        if (i915_request_wait(rq, 0, timeout) < 0)
 122                err = -ETIME;
 123        i915_request_put(rq);
 124
 125        rmb(); /* We know the request is written, make sure all state is too! */
 126        return err;
 127}
 128
 129static int live_lrc_layout(void *arg)
 130{
 131        struct intel_gt *gt = arg;
 132        struct intel_engine_cs *engine;
 133        enum intel_engine_id id;
 134        u32 *lrc;
 135        int err;
 136
 137        /*
 138         * Check the registers offsets we use to create the initial reg state
 139         * match the layout saved by HW.
 140         */
 141
 142        lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
 143        if (!lrc)
 144                return -ENOMEM;
 145        GEM_BUG_ON(offset_in_page(lrc));
 146
 147        err = 0;
 148        for_each_engine(engine, gt, id) {
 149                u32 *hw;
 150                int dw;
 151
 152                if (!engine->default_state)
 153                        continue;
 154
 155                hw = shmem_pin_map(engine->default_state);
 156                if (IS_ERR(hw)) {
 157                        err = PTR_ERR(hw);
 158                        break;
 159                }
 160                hw += LRC_STATE_OFFSET / sizeof(*hw);
 161
 162                __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
 163                                engine->kernel_context, engine, true);
 164
 165                dw = 0;
 166                do {
 167                        u32 lri = READ_ONCE(hw[dw]);
 168
 169                        if (lri == 0) {
 170                                dw++;
 171                                continue;
 172                        }
 173
 174                        if (lrc[dw] == 0) {
 175                                pr_debug("%s: skipped instruction %x at dword %d\n",
 176                                         engine->name, lri, dw);
 177                                dw++;
 178                                continue;
 179                        }
 180
 181                        if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 182                                pr_err("%s: Expected LRI command at dword %d, found %08x\n",
 183                                       engine->name, dw, lri);
 184                                err = -EINVAL;
 185                                break;
 186                        }
 187
 188                        if (lrc[dw] != lri) {
 189                                pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
 190                                       engine->name, dw, lri, lrc[dw]);
 191                                err = -EINVAL;
 192                                break;
 193                        }
 194
 195                        lri &= 0x7f;
 196                        lri++;
 197                        dw++;
 198
 199                        while (lri) {
 200                                u32 offset = READ_ONCE(hw[dw]);
 201
 202                                if (offset != lrc[dw]) {
 203                                        pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
 204                                               engine->name, dw, offset, lrc[dw]);
 205                                        err = -EINVAL;
 206                                        break;
 207                                }
 208
 209                                /*
 210                                 * Skip over the actual register value as we
 211                                 * expect that to differ.
 212                                 */
 213                                dw += 2;
 214                                lri -= 2;
 215                        }
 216                } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 217
 218                if (err) {
 219                        pr_info("%s: HW register image:\n", engine->name);
 220                        igt_hexdump(hw, PAGE_SIZE);
 221
 222                        pr_info("%s: SW register image:\n", engine->name);
 223                        igt_hexdump(lrc, PAGE_SIZE);
 224                }
 225
 226                shmem_unpin_map(engine->default_state, hw);
 227                if (err)
 228                        break;
 229        }
 230
 231        free_page((unsigned long)lrc);
 232        return err;
 233}
 234
 235static int find_offset(const u32 *lri, u32 offset)
 236{
 237        int i;
 238
 239        for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
 240                if (lri[i] == offset)
 241                        return i;
 242
 243        return -1;
 244}
 245
 246static int live_lrc_fixed(void *arg)
 247{
 248        struct intel_gt *gt = arg;
 249        struct intel_engine_cs *engine;
 250        enum intel_engine_id id;
 251        int err = 0;
 252
 253        /*
 254         * Check the assumed register offsets match the actual locations in
 255         * the context image.
 256         */
 257
 258        for_each_engine(engine, gt, id) {
 259                const struct {
 260                        u32 reg;
 261                        u32 offset;
 262                        const char *name;
 263                } tbl[] = {
 264                        {
 265                                i915_mmio_reg_offset(RING_START(engine->mmio_base)),
 266                                CTX_RING_START - 1,
 267                                "RING_START"
 268                        },
 269                        {
 270                                i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
 271                                CTX_RING_CTL - 1,
 272                                "RING_CTL"
 273                        },
 274                        {
 275                                i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
 276                                CTX_RING_HEAD - 1,
 277                                "RING_HEAD"
 278                        },
 279                        {
 280                                i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
 281                                CTX_RING_TAIL - 1,
 282                                "RING_TAIL"
 283                        },
 284                        {
 285                                i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
 286                                lrc_ring_mi_mode(engine),
 287                                "RING_MI_MODE"
 288                        },
 289                        {
 290                                i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
 291                                CTX_BB_STATE - 1,
 292                                "BB_STATE"
 293                        },
 294                        {
 295                                i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
 296                                lrc_ring_wa_bb_per_ctx(engine),
 297                                "RING_BB_PER_CTX_PTR"
 298                        },
 299                        {
 300                                i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
 301                                lrc_ring_indirect_ptr(engine),
 302                                "RING_INDIRECT_CTX_PTR"
 303                        },
 304                        {
 305                                i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
 306                                lrc_ring_indirect_offset(engine),
 307                                "RING_INDIRECT_CTX_OFFSET"
 308                        },
 309                        {
 310                                i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
 311                                CTX_TIMESTAMP - 1,
 312                                "RING_CTX_TIMESTAMP"
 313                        },
 314                        {
 315                                i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
 316                                lrc_ring_gpr0(engine),
 317                                "RING_CS_GPR0"
 318                        },
 319                        {
 320                                i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
 321                                lrc_ring_cmd_buf_cctl(engine),
 322                                "RING_CMD_BUF_CCTL"
 323                        },
 324                        { },
 325                }, *t;
 326                u32 *hw;
 327
 328                if (!engine->default_state)
 329                        continue;
 330
 331                hw = shmem_pin_map(engine->default_state);
 332                if (IS_ERR(hw)) {
 333                        err = PTR_ERR(hw);
 334                        break;
 335                }
 336                hw += LRC_STATE_OFFSET / sizeof(*hw);
 337
 338                for (t = tbl; t->name; t++) {
 339                        int dw = find_offset(hw, t->reg);
 340
 341                        if (dw != t->offset) {
 342                                pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
 343                                       engine->name,
 344                                       t->name,
 345                                       t->reg,
 346                                       dw,
 347                                       t->offset);
 348                                err = -EINVAL;
 349                        }
 350                }
 351
 352                shmem_unpin_map(engine->default_state, hw);
 353        }
 354
 355        return err;
 356}
 357
 358static int __live_lrc_state(struct intel_engine_cs *engine,
 359                            struct i915_vma *scratch)
 360{
 361        struct intel_context *ce;
 362        struct i915_request *rq;
 363        struct i915_gem_ww_ctx ww;
 364        enum {
 365                RING_START_IDX = 0,
 366                RING_TAIL_IDX,
 367                MAX_IDX
 368        };
 369        u32 expected[MAX_IDX];
 370        u32 *cs;
 371        int err;
 372        int n;
 373
 374        ce = intel_context_create(engine);
 375        if (IS_ERR(ce))
 376                return PTR_ERR(ce);
 377
 378        i915_gem_ww_ctx_init(&ww, false);
 379retry:
 380        err = i915_gem_object_lock(scratch->obj, &ww);
 381        if (!err)
 382                err = intel_context_pin_ww(ce, &ww);
 383        if (err)
 384                goto err_put;
 385
 386        rq = i915_request_create(ce);
 387        if (IS_ERR(rq)) {
 388                err = PTR_ERR(rq);
 389                goto err_unpin;
 390        }
 391
 392        cs = intel_ring_begin(rq, 4 * MAX_IDX);
 393        if (IS_ERR(cs)) {
 394                err = PTR_ERR(cs);
 395                i915_request_add(rq);
 396                goto err_unpin;
 397        }
 398
 399        *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 400        *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
 401        *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
 402        *cs++ = 0;
 403
 404        expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
 405
 406        *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 407        *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
 408        *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
 409        *cs++ = 0;
 410
 411        err = i915_request_await_object(rq, scratch->obj, true);
 412        if (!err)
 413                err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 414
 415        i915_request_get(rq);
 416        i915_request_add(rq);
 417        if (err)
 418                goto err_rq;
 419
 420        intel_engine_flush_submission(engine);
 421        expected[RING_TAIL_IDX] = ce->ring->tail;
 422
 423        if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 424                err = -ETIME;
 425                goto err_rq;
 426        }
 427
 428        cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 429        if (IS_ERR(cs)) {
 430                err = PTR_ERR(cs);
 431                goto err_rq;
 432        }
 433
 434        for (n = 0; n < MAX_IDX; n++) {
 435                if (cs[n] != expected[n]) {
 436                        pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
 437                               engine->name, n, cs[n], expected[n]);
 438                        err = -EINVAL;
 439                        break;
 440                }
 441        }
 442
 443        i915_gem_object_unpin_map(scratch->obj);
 444
 445err_rq:
 446        i915_request_put(rq);
 447err_unpin:
 448        intel_context_unpin(ce);
 449err_put:
 450        if (err == -EDEADLK) {
 451                err = i915_gem_ww_ctx_backoff(&ww);
 452                if (!err)
 453                        goto retry;
 454        }
 455        i915_gem_ww_ctx_fini(&ww);
 456        intel_context_put(ce);
 457        return err;
 458}
 459
 460static int live_lrc_state(void *arg)
 461{
 462        struct intel_gt *gt = arg;
 463        struct intel_engine_cs *engine;
 464        struct i915_vma *scratch;
 465        enum intel_engine_id id;
 466        int err = 0;
 467
 468        /*
 469         * Check the live register state matches what we expect for this
 470         * intel_context.
 471         */
 472
 473        scratch = create_scratch(gt);
 474        if (IS_ERR(scratch))
 475                return PTR_ERR(scratch);
 476
 477        for_each_engine(engine, gt, id) {
 478                err = __live_lrc_state(engine, scratch);
 479                if (err)
 480                        break;
 481        }
 482
 483        if (igt_flush_test(gt->i915))
 484                err = -EIO;
 485
 486        i915_vma_unpin_and_release(&scratch, 0);
 487        return err;
 488}
 489
 490static int gpr_make_dirty(struct intel_context *ce)
 491{
 492        struct i915_request *rq;
 493        u32 *cs;
 494        int n;
 495
 496        rq = intel_context_create_request(ce);
 497        if (IS_ERR(rq))
 498                return PTR_ERR(rq);
 499
 500        cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
 501        if (IS_ERR(cs)) {
 502                i915_request_add(rq);
 503                return PTR_ERR(cs);
 504        }
 505
 506        *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
 507        for (n = 0; n < NUM_GPR_DW; n++) {
 508                *cs++ = CS_GPR(ce->engine, n);
 509                *cs++ = STACK_MAGIC;
 510        }
 511        *cs++ = MI_NOOP;
 512
 513        intel_ring_advance(rq, cs);
 514
 515        rq->sched.attr.priority = I915_PRIORITY_BARRIER;
 516        i915_request_add(rq);
 517
 518        return 0;
 519}
 520
 521static struct i915_request *
 522__gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
 523{
 524        const u32 offset =
 525                i915_ggtt_offset(ce->engine->status_page.vma) +
 526                offset_in_page(slot);
 527        struct i915_request *rq;
 528        u32 *cs;
 529        int err;
 530        int n;
 531
 532        rq = intel_context_create_request(ce);
 533        if (IS_ERR(rq))
 534                return rq;
 535
 536        cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
 537        if (IS_ERR(cs)) {
 538                i915_request_add(rq);
 539                return ERR_CAST(cs);
 540        }
 541
 542        *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 543        *cs++ = MI_NOOP;
 544
 545        *cs++ = MI_SEMAPHORE_WAIT |
 546                MI_SEMAPHORE_GLOBAL_GTT |
 547                MI_SEMAPHORE_POLL |
 548                MI_SEMAPHORE_SAD_NEQ_SDD;
 549        *cs++ = 0;
 550        *cs++ = offset;
 551        *cs++ = 0;
 552
 553        for (n = 0; n < NUM_GPR_DW; n++) {
 554                *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 555                *cs++ = CS_GPR(ce->engine, n);
 556                *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
 557                *cs++ = 0;
 558        }
 559
 560        i915_vma_lock(scratch);
 561        err = i915_request_await_object(rq, scratch->obj, true);
 562        if (!err)
 563                err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
 564        i915_vma_unlock(scratch);
 565
 566        i915_request_get(rq);
 567        i915_request_add(rq);
 568        if (err) {
 569                i915_request_put(rq);
 570                rq = ERR_PTR(err);
 571        }
 572
 573        return rq;
 574}
 575
 576static int __live_lrc_gpr(struct intel_engine_cs *engine,
 577                          struct i915_vma *scratch,
 578                          bool preempt)
 579{
 580        u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
 581        struct intel_context *ce;
 582        struct i915_request *rq;
 583        u32 *cs;
 584        int err;
 585        int n;
 586
 587        if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
 588                return 0; /* GPR only on rcs0 for gen8 */
 589
 590        err = gpr_make_dirty(engine->kernel_context);
 591        if (err)
 592                return err;
 593
 594        ce = intel_context_create(engine);
 595        if (IS_ERR(ce))
 596                return PTR_ERR(ce);
 597
 598        rq = __gpr_read(ce, scratch, slot);
 599        if (IS_ERR(rq)) {
 600                err = PTR_ERR(rq);
 601                goto err_put;
 602        }
 603
 604        err = wait_for_submit(engine, rq, HZ / 2);
 605        if (err)
 606                goto err_rq;
 607
 608        if (preempt) {
 609                err = gpr_make_dirty(engine->kernel_context);
 610                if (err)
 611                        goto err_rq;
 612
 613                err = emit_semaphore_signal(engine->kernel_context, slot);
 614                if (err)
 615                        goto err_rq;
 616
 617                err = wait_for_submit(engine, rq, HZ / 2);
 618                if (err)
 619                        goto err_rq;
 620        } else {
 621                slot[0] = 1;
 622                wmb();
 623        }
 624
 625        if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 626                err = -ETIME;
 627                goto err_rq;
 628        }
 629
 630        cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
 631        if (IS_ERR(cs)) {
 632                err = PTR_ERR(cs);
 633                goto err_rq;
 634        }
 635
 636        for (n = 0; n < NUM_GPR_DW; n++) {
 637                if (cs[n]) {
 638                        pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
 639                               engine->name,
 640                               n / 2, n & 1 ? "udw" : "ldw",
 641                               cs[n]);
 642                        err = -EINVAL;
 643                        break;
 644                }
 645        }
 646
 647        i915_gem_object_unpin_map(scratch->obj);
 648
 649err_rq:
 650        memset32(&slot[0], -1, 4);
 651        wmb();
 652        i915_request_put(rq);
 653err_put:
 654        intel_context_put(ce);
 655        return err;
 656}
 657
 658static int live_lrc_gpr(void *arg)
 659{
 660        struct intel_gt *gt = arg;
 661        struct intel_engine_cs *engine;
 662        struct i915_vma *scratch;
 663        enum intel_engine_id id;
 664        int err = 0;
 665
 666        /*
 667         * Check that GPR registers are cleared in new contexts as we need
 668         * to avoid leaking any information from previous contexts.
 669         */
 670
 671        scratch = create_scratch(gt);
 672        if (IS_ERR(scratch))
 673                return PTR_ERR(scratch);
 674
 675        for_each_engine(engine, gt, id) {
 676                st_engine_heartbeat_disable(engine);
 677
 678                err = __live_lrc_gpr(engine, scratch, false);
 679                if (err)
 680                        goto err;
 681
 682                err = __live_lrc_gpr(engine, scratch, true);
 683                if (err)
 684                        goto err;
 685
 686err:
 687                st_engine_heartbeat_enable(engine);
 688                if (igt_flush_test(gt->i915))
 689                        err = -EIO;
 690                if (err)
 691                        break;
 692        }
 693
 694        i915_vma_unpin_and_release(&scratch, 0);
 695        return err;
 696}
 697
 698static struct i915_request *
 699create_timestamp(struct intel_context *ce, void *slot, int idx)
 700{
 701        const u32 offset =
 702                i915_ggtt_offset(ce->engine->status_page.vma) +
 703                offset_in_page(slot);
 704        struct i915_request *rq;
 705        u32 *cs;
 706        int err;
 707
 708        rq = intel_context_create_request(ce);
 709        if (IS_ERR(rq))
 710                return rq;
 711
 712        cs = intel_ring_begin(rq, 10);
 713        if (IS_ERR(cs)) {
 714                err = PTR_ERR(cs);
 715                goto err;
 716        }
 717
 718        *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 719        *cs++ = MI_NOOP;
 720
 721        *cs++ = MI_SEMAPHORE_WAIT |
 722                MI_SEMAPHORE_GLOBAL_GTT |
 723                MI_SEMAPHORE_POLL |
 724                MI_SEMAPHORE_SAD_NEQ_SDD;
 725        *cs++ = 0;
 726        *cs++ = offset;
 727        *cs++ = 0;
 728
 729        *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
 730        *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
 731        *cs++ = offset + idx * sizeof(u32);
 732        *cs++ = 0;
 733
 734        intel_ring_advance(rq, cs);
 735
 736        err = 0;
 737err:
 738        i915_request_get(rq);
 739        i915_request_add(rq);
 740        if (err) {
 741                i915_request_put(rq);
 742                return ERR_PTR(err);
 743        }
 744
 745        return rq;
 746}
 747
 748struct lrc_timestamp {
 749        struct intel_engine_cs *engine;
 750        struct intel_context *ce[2];
 751        u32 poison;
 752};
 753
 754static bool timestamp_advanced(u32 start, u32 end)
 755{
 756        return (s32)(end - start) > 0;
 757}
 758
 759static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
 760{
 761        u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
 762        struct i915_request *rq;
 763        u32 timestamp;
 764        int err = 0;
 765
 766        arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
 767        rq = create_timestamp(arg->ce[0], slot, 1);
 768        if (IS_ERR(rq))
 769                return PTR_ERR(rq);
 770
 771        err = wait_for_submit(rq->engine, rq, HZ / 2);
 772        if (err)
 773                goto err;
 774
 775        if (preempt) {
 776                arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
 777                err = emit_semaphore_signal(arg->ce[1], slot);
 778                if (err)
 779                        goto err;
 780        } else {
 781                slot[0] = 1;
 782                wmb();
 783        }
 784
 785        /* And wait for switch to kernel (to save our context to memory) */
 786        err = context_flush(arg->ce[0], HZ / 2);
 787        if (err)
 788                goto err;
 789
 790        if (!timestamp_advanced(arg->poison, slot[1])) {
 791                pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
 792                       arg->engine->name, preempt ? "preempt" : "simple",
 793                       arg->poison, slot[1]);
 794                err = -EINVAL;
 795        }
 796
 797        timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
 798        if (!timestamp_advanced(slot[1], timestamp)) {
 799                pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
 800                       arg->engine->name, preempt ? "preempt" : "simple",
 801                       slot[1], timestamp);
 802                err = -EINVAL;
 803        }
 804
 805err:
 806        memset32(slot, -1, 4);
 807        i915_request_put(rq);
 808        return err;
 809}
 810
 811static int live_lrc_timestamp(void *arg)
 812{
 813        struct lrc_timestamp data = {};
 814        struct intel_gt *gt = arg;
 815        enum intel_engine_id id;
 816        const u32 poison[] = {
 817                0,
 818                S32_MAX,
 819                (u32)S32_MAX + 1,
 820                U32_MAX,
 821        };
 822
 823        /*
 824         * We want to verify that the timestamp is saved and restore across
 825         * context switches and is monotonic.
 826         *
 827         * So we do this with a little bit of LRC poisoning to check various
 828         * boundary conditions, and see what happens if we preempt the context
 829         * with a second request (carrying more poison into the timestamp).
 830         */
 831
 832        for_each_engine(data.engine, gt, id) {
 833                int i, err = 0;
 834
 835                st_engine_heartbeat_disable(data.engine);
 836
 837                for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 838                        struct intel_context *tmp;
 839
 840                        tmp = intel_context_create(data.engine);
 841                        if (IS_ERR(tmp)) {
 842                                err = PTR_ERR(tmp);
 843                                goto err;
 844                        }
 845
 846                        err = intel_context_pin(tmp);
 847                        if (err) {
 848                                intel_context_put(tmp);
 849                                goto err;
 850                        }
 851
 852                        data.ce[i] = tmp;
 853                }
 854
 855                for (i = 0; i < ARRAY_SIZE(poison); i++) {
 856                        data.poison = poison[i];
 857
 858                        err = __lrc_timestamp(&data, false);
 859                        if (err)
 860                                break;
 861
 862                        err = __lrc_timestamp(&data, true);
 863                        if (err)
 864                                break;
 865                }
 866
 867err:
 868                st_engine_heartbeat_enable(data.engine);
 869                for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 870                        if (!data.ce[i])
 871                                break;
 872
 873                        intel_context_unpin(data.ce[i]);
 874                        intel_context_put(data.ce[i]);
 875                }
 876
 877                if (igt_flush_test(gt->i915))
 878                        err = -EIO;
 879                if (err)
 880                        return err;
 881        }
 882
 883        return 0;
 884}
 885
 886static struct i915_vma *
 887create_user_vma(struct i915_address_space *vm, unsigned long size)
 888{
 889        struct drm_i915_gem_object *obj;
 890        struct i915_vma *vma;
 891        int err;
 892
 893        obj = i915_gem_object_create_internal(vm->i915, size);
 894        if (IS_ERR(obj))
 895                return ERR_CAST(obj);
 896
 897        vma = i915_vma_instance(obj, vm, NULL);
 898        if (IS_ERR(vma)) {
 899                i915_gem_object_put(obj);
 900                return vma;
 901        }
 902
 903        err = i915_vma_pin(vma, 0, 0, PIN_USER);
 904        if (err) {
 905                i915_gem_object_put(obj);
 906                return ERR_PTR(err);
 907        }
 908
 909        return vma;
 910}
 911
 912static struct i915_vma *
 913store_context(struct intel_context *ce, struct i915_vma *scratch)
 914{
 915        struct i915_vma *batch;
 916        u32 dw, x, *cs, *hw;
 917        u32 *defaults;
 918
 919        batch = create_user_vma(ce->vm, SZ_64K);
 920        if (IS_ERR(batch))
 921                return batch;
 922
 923        cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
 924        if (IS_ERR(cs)) {
 925                i915_vma_put(batch);
 926                return ERR_CAST(cs);
 927        }
 928
 929        defaults = shmem_pin_map(ce->engine->default_state);
 930        if (!defaults) {
 931                i915_gem_object_unpin_map(batch->obj);
 932                i915_vma_put(batch);
 933                return ERR_PTR(-ENOMEM);
 934        }
 935
 936        x = 0;
 937        dw = 0;
 938        hw = defaults;
 939        hw += LRC_STATE_OFFSET / sizeof(*hw);
 940        do {
 941                u32 len = hw[dw] & 0x7f;
 942
 943                if (hw[dw] == 0) {
 944                        dw++;
 945                        continue;
 946                }
 947
 948                if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
 949                        dw += len + 2;
 950                        continue;
 951                }
 952
 953                dw++;
 954                len = (len + 1) / 2;
 955                while (len--) {
 956                        *cs++ = MI_STORE_REGISTER_MEM_GEN8;
 957                        *cs++ = hw[dw];
 958                        *cs++ = lower_32_bits(scratch->node.start + x);
 959                        *cs++ = upper_32_bits(scratch->node.start + x);
 960
 961                        dw += 2;
 962                        x += 4;
 963                }
 964        } while (dw < PAGE_SIZE / sizeof(u32) &&
 965                 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
 966
 967        *cs++ = MI_BATCH_BUFFER_END;
 968
 969        shmem_unpin_map(ce->engine->default_state, defaults);
 970
 971        i915_gem_object_flush_map(batch->obj);
 972        i915_gem_object_unpin_map(batch->obj);
 973
 974        return batch;
 975}
 976
 977static int move_to_active(struct i915_request *rq,
 978                          struct i915_vma *vma,
 979                          unsigned int flags)
 980{
 981        int err;
 982
 983        i915_vma_lock(vma);
 984        err = i915_request_await_object(rq, vma->obj, flags);
 985        if (!err)
 986                err = i915_vma_move_to_active(vma, rq, flags);
 987        i915_vma_unlock(vma);
 988
 989        return err;
 990}
 991
 992static struct i915_request *
 993record_registers(struct intel_context *ce,
 994                 struct i915_vma *before,
 995                 struct i915_vma *after,
 996                 u32 *sema)
 997{
 998        struct i915_vma *b_before, *b_after;
 999        struct i915_request *rq;
1000        u32 *cs;
1001        int err;
1002
1003        b_before = store_context(ce, before);
1004        if (IS_ERR(b_before))
1005                return ERR_CAST(b_before);
1006
1007        b_after = store_context(ce, after);
1008        if (IS_ERR(b_after)) {
1009                rq = ERR_CAST(b_after);
1010                goto err_before;
1011        }
1012
1013        rq = intel_context_create_request(ce);
1014        if (IS_ERR(rq))
1015                goto err_after;
1016
1017        err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1018        if (err)
1019                goto err_rq;
1020
1021        err = move_to_active(rq, b_before, 0);
1022        if (err)
1023                goto err_rq;
1024
1025        err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1026        if (err)
1027                goto err_rq;
1028
1029        err = move_to_active(rq, b_after, 0);
1030        if (err)
1031                goto err_rq;
1032
1033        cs = intel_ring_begin(rq, 14);
1034        if (IS_ERR(cs)) {
1035                err = PTR_ERR(cs);
1036                goto err_rq;
1037        }
1038
1039        *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1040        *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1041        *cs++ = lower_32_bits(b_before->node.start);
1042        *cs++ = upper_32_bits(b_before->node.start);
1043
1044        *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1045        *cs++ = MI_SEMAPHORE_WAIT |
1046                MI_SEMAPHORE_GLOBAL_GTT |
1047                MI_SEMAPHORE_POLL |
1048                MI_SEMAPHORE_SAD_NEQ_SDD;
1049        *cs++ = 0;
1050        *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1051                offset_in_page(sema);
1052        *cs++ = 0;
1053        *cs++ = MI_NOOP;
1054
1055        *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1056        *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1057        *cs++ = lower_32_bits(b_after->node.start);
1058        *cs++ = upper_32_bits(b_after->node.start);
1059
1060        intel_ring_advance(rq, cs);
1061
1062        WRITE_ONCE(*sema, 0);
1063        i915_request_get(rq);
1064        i915_request_add(rq);
1065err_after:
1066        i915_vma_put(b_after);
1067err_before:
1068        i915_vma_put(b_before);
1069        return rq;
1070
1071err_rq:
1072        i915_request_add(rq);
1073        rq = ERR_PTR(err);
1074        goto err_after;
1075}
1076
1077static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1078{
1079        struct i915_vma *batch;
1080        u32 dw, *cs, *hw;
1081        u32 *defaults;
1082
1083        batch = create_user_vma(ce->vm, SZ_64K);
1084        if (IS_ERR(batch))
1085                return batch;
1086
1087        cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1088        if (IS_ERR(cs)) {
1089                i915_vma_put(batch);
1090                return ERR_CAST(cs);
1091        }
1092
1093        defaults = shmem_pin_map(ce->engine->default_state);
1094        if (!defaults) {
1095                i915_gem_object_unpin_map(batch->obj);
1096                i915_vma_put(batch);
1097                return ERR_PTR(-ENOMEM);
1098        }
1099
1100        dw = 0;
1101        hw = defaults;
1102        hw += LRC_STATE_OFFSET / sizeof(*hw);
1103        do {
1104                u32 len = hw[dw] & 0x7f;
1105
1106                if (hw[dw] == 0) {
1107                        dw++;
1108                        continue;
1109                }
1110
1111                if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1112                        dw += len + 2;
1113                        continue;
1114                }
1115
1116                dw++;
1117                len = (len + 1) / 2;
1118                *cs++ = MI_LOAD_REGISTER_IMM(len);
1119                while (len--) {
1120                        *cs++ = hw[dw];
1121                        *cs++ = poison;
1122                        dw += 2;
1123                }
1124        } while (dw < PAGE_SIZE / sizeof(u32) &&
1125                 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1126
1127        *cs++ = MI_BATCH_BUFFER_END;
1128
1129        shmem_unpin_map(ce->engine->default_state, defaults);
1130
1131        i915_gem_object_flush_map(batch->obj);
1132        i915_gem_object_unpin_map(batch->obj);
1133
1134        return batch;
1135}
1136
1137static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1138{
1139        struct i915_request *rq;
1140        struct i915_vma *batch;
1141        u32 *cs;
1142        int err;
1143
1144        batch = load_context(ce, poison);
1145        if (IS_ERR(batch))
1146                return PTR_ERR(batch);
1147
1148        rq = intel_context_create_request(ce);
1149        if (IS_ERR(rq)) {
1150                err = PTR_ERR(rq);
1151                goto err_batch;
1152        }
1153
1154        err = move_to_active(rq, batch, 0);
1155        if (err)
1156                goto err_rq;
1157
1158        cs = intel_ring_begin(rq, 8);
1159        if (IS_ERR(cs)) {
1160                err = PTR_ERR(cs);
1161                goto err_rq;
1162        }
1163
1164        *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1165        *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1166        *cs++ = lower_32_bits(batch->node.start);
1167        *cs++ = upper_32_bits(batch->node.start);
1168
1169        *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1170        *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1171                offset_in_page(sema);
1172        *cs++ = 0;
1173        *cs++ = 1;
1174
1175        intel_ring_advance(rq, cs);
1176
1177        rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1178err_rq:
1179        i915_request_add(rq);
1180err_batch:
1181        i915_vma_put(batch);
1182        return err;
1183}
1184
1185static bool is_moving(u32 a, u32 b)
1186{
1187        return a != b;
1188}
1189
1190static int compare_isolation(struct intel_engine_cs *engine,
1191                             struct i915_vma *ref[2],
1192                             struct i915_vma *result[2],
1193                             struct intel_context *ce,
1194                             u32 poison)
1195{
1196        u32 x, dw, *hw, *lrc;
1197        u32 *A[2], *B[2];
1198        u32 *defaults;
1199        int err = 0;
1200
1201        A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1202        if (IS_ERR(A[0]))
1203                return PTR_ERR(A[0]);
1204
1205        A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1206        if (IS_ERR(A[1])) {
1207                err = PTR_ERR(A[1]);
1208                goto err_A0;
1209        }
1210
1211        B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1212        if (IS_ERR(B[0])) {
1213                err = PTR_ERR(B[0]);
1214                goto err_A1;
1215        }
1216
1217        B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1218        if (IS_ERR(B[1])) {
1219                err = PTR_ERR(B[1]);
1220                goto err_B0;
1221        }
1222
1223        lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1224                                               i915_coherent_map_type(engine->i915,
1225                                                                      ce->state->obj,
1226                                                                      false));
1227        if (IS_ERR(lrc)) {
1228                err = PTR_ERR(lrc);
1229                goto err_B1;
1230        }
1231        lrc += LRC_STATE_OFFSET / sizeof(*hw);
1232
1233        defaults = shmem_pin_map(ce->engine->default_state);
1234        if (!defaults) {
1235                err = -ENOMEM;
1236                goto err_lrc;
1237        }
1238
1239        x = 0;
1240        dw = 0;
1241        hw = defaults;
1242        hw += LRC_STATE_OFFSET / sizeof(*hw);
1243        do {
1244                u32 len = hw[dw] & 0x7f;
1245
1246                if (hw[dw] == 0) {
1247                        dw++;
1248                        continue;
1249                }
1250
1251                if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1252                        dw += len + 2;
1253                        continue;
1254                }
1255
1256                dw++;
1257                len = (len + 1) / 2;
1258                while (len--) {
1259                        if (!is_moving(A[0][x], A[1][x]) &&
1260                            (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1261                                switch (hw[dw] & 4095) {
1262                                case 0x30: /* RING_HEAD */
1263                                case 0x34: /* RING_TAIL */
1264                                        break;
1265
1266                                default:
1267                                        pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1268                                               engine->name, dw,
1269                                               hw[dw], hw[dw + 1],
1270                                               A[0][x], B[0][x], B[1][x],
1271                                               poison, lrc[dw + 1]);
1272                                        err = -EINVAL;
1273                                }
1274                        }
1275                        dw += 2;
1276                        x++;
1277                }
1278        } while (dw < PAGE_SIZE / sizeof(u32) &&
1279                 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1280
1281        shmem_unpin_map(ce->engine->default_state, defaults);
1282err_lrc:
1283        i915_gem_object_unpin_map(ce->state->obj);
1284err_B1:
1285        i915_gem_object_unpin_map(result[1]->obj);
1286err_B0:
1287        i915_gem_object_unpin_map(result[0]->obj);
1288err_A1:
1289        i915_gem_object_unpin_map(ref[1]->obj);
1290err_A0:
1291        i915_gem_object_unpin_map(ref[0]->obj);
1292        return err;
1293}
1294
1295static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1296{
1297        u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1298        struct i915_vma *ref[2], *result[2];
1299        struct intel_context *A, *B;
1300        struct i915_request *rq;
1301        int err;
1302
1303        A = intel_context_create(engine);
1304        if (IS_ERR(A))
1305                return PTR_ERR(A);
1306
1307        B = intel_context_create(engine);
1308        if (IS_ERR(B)) {
1309                err = PTR_ERR(B);
1310                goto err_A;
1311        }
1312
1313        ref[0] = create_user_vma(A->vm, SZ_64K);
1314        if (IS_ERR(ref[0])) {
1315                err = PTR_ERR(ref[0]);
1316                goto err_B;
1317        }
1318
1319        ref[1] = create_user_vma(A->vm, SZ_64K);
1320        if (IS_ERR(ref[1])) {
1321                err = PTR_ERR(ref[1]);
1322                goto err_ref0;
1323        }
1324
1325        rq = record_registers(A, ref[0], ref[1], sema);
1326        if (IS_ERR(rq)) {
1327                err = PTR_ERR(rq);
1328                goto err_ref1;
1329        }
1330
1331        WRITE_ONCE(*sema, 1);
1332        wmb();
1333
1334        if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1335                i915_request_put(rq);
1336                err = -ETIME;
1337                goto err_ref1;
1338        }
1339        i915_request_put(rq);
1340
1341        result[0] = create_user_vma(A->vm, SZ_64K);
1342        if (IS_ERR(result[0])) {
1343                err = PTR_ERR(result[0]);
1344                goto err_ref1;
1345        }
1346
1347        result[1] = create_user_vma(A->vm, SZ_64K);
1348        if (IS_ERR(result[1])) {
1349                err = PTR_ERR(result[1]);
1350                goto err_result0;
1351        }
1352
1353        rq = record_registers(A, result[0], result[1], sema);
1354        if (IS_ERR(rq)) {
1355                err = PTR_ERR(rq);
1356                goto err_result1;
1357        }
1358
1359        err = poison_registers(B, poison, sema);
1360        if (err) {
1361                WRITE_ONCE(*sema, -1);
1362                i915_request_put(rq);
1363                goto err_result1;
1364        }
1365
1366        if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1367                i915_request_put(rq);
1368                err = -ETIME;
1369                goto err_result1;
1370        }
1371        i915_request_put(rq);
1372
1373        err = compare_isolation(engine, ref, result, A, poison);
1374
1375err_result1:
1376        i915_vma_put(result[1]);
1377err_result0:
1378        i915_vma_put(result[0]);
1379err_ref1:
1380        i915_vma_put(ref[1]);
1381err_ref0:
1382        i915_vma_put(ref[0]);
1383err_B:
1384        intel_context_put(B);
1385err_A:
1386        intel_context_put(A);
1387        return err;
1388}
1389
1390static bool skip_isolation(const struct intel_engine_cs *engine)
1391{
1392        if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1393                return true;
1394
1395        if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1396                return true;
1397
1398        return false;
1399}
1400
1401static int live_lrc_isolation(void *arg)
1402{
1403        struct intel_gt *gt = arg;
1404        struct intel_engine_cs *engine;
1405        enum intel_engine_id id;
1406        const u32 poison[] = {
1407                STACK_MAGIC,
1408                0x3a3a3a3a,
1409                0x5c5c5c5c,
1410                0xffffffff,
1411                0xffff0000,
1412        };
1413        int err = 0;
1414
1415        /*
1416         * Our goal is try and verify that per-context state cannot be
1417         * tampered with by another non-privileged client.
1418         *
1419         * We take the list of context registers from the LRI in the default
1420         * context image and attempt to modify that list from a remote context.
1421         */
1422
1423        for_each_engine(engine, gt, id) {
1424                int i;
1425
1426                /* Just don't even ask */
1427                if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1428                    skip_isolation(engine))
1429                        continue;
1430
1431                intel_engine_pm_get(engine);
1432                for (i = 0; i < ARRAY_SIZE(poison); i++) {
1433                        int result;
1434
1435                        result = __lrc_isolation(engine, poison[i]);
1436                        if (result && !err)
1437                                err = result;
1438
1439                        result = __lrc_isolation(engine, ~poison[i]);
1440                        if (result && !err)
1441                                err = result;
1442                }
1443                intel_engine_pm_put(engine);
1444                if (igt_flush_test(gt->i915)) {
1445                        err = -EIO;
1446                        break;
1447                }
1448        }
1449
1450        return err;
1451}
1452
1453static int indirect_ctx_submit_req(struct intel_context *ce)
1454{
1455        struct i915_request *rq;
1456        int err = 0;
1457
1458        rq = intel_context_create_request(ce);
1459        if (IS_ERR(rq))
1460                return PTR_ERR(rq);
1461
1462        i915_request_get(rq);
1463        i915_request_add(rq);
1464
1465        if (i915_request_wait(rq, 0, HZ / 5) < 0)
1466                err = -ETIME;
1467
1468        i915_request_put(rq);
1469
1470        return err;
1471}
1472
1473#define CTX_BB_CANARY_OFFSET (3 * 1024)
1474#define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1475
1476static u32 *
1477emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1478{
1479        *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1480                MI_SRM_LRM_GLOBAL_GTT |
1481                MI_LRI_LRM_CS_MMIO;
1482        *cs++ = i915_mmio_reg_offset(RING_START(0));
1483        *cs++ = i915_ggtt_offset(ce->state) +
1484                context_wa_bb_offset(ce) +
1485                CTX_BB_CANARY_OFFSET;
1486        *cs++ = 0;
1487
1488        return cs;
1489}
1490
1491static void
1492indirect_ctx_bb_setup(struct intel_context *ce)
1493{
1494        u32 *cs = context_indirect_bb(ce);
1495
1496        cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1497
1498        setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1499}
1500
1501static bool check_ring_start(struct intel_context *ce)
1502{
1503        const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1504                LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1505
1506        if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1507                return true;
1508
1509        pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1510               ctx_bb[CTX_BB_CANARY_INDEX],
1511               ce->lrc_reg_state[CTX_RING_START]);
1512
1513        return false;
1514}
1515
1516static int indirect_ctx_bb_check(struct intel_context *ce)
1517{
1518        int err;
1519
1520        err = indirect_ctx_submit_req(ce);
1521        if (err)
1522                return err;
1523
1524        if (!check_ring_start(ce))
1525                return -EINVAL;
1526
1527        return 0;
1528}
1529
1530static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1531{
1532        struct intel_context *a, *b;
1533        int err;
1534
1535        a = intel_context_create(engine);
1536        if (IS_ERR(a))
1537                return PTR_ERR(a);
1538        err = intel_context_pin(a);
1539        if (err)
1540                goto put_a;
1541
1542        b = intel_context_create(engine);
1543        if (IS_ERR(b)) {
1544                err = PTR_ERR(b);
1545                goto unpin_a;
1546        }
1547        err = intel_context_pin(b);
1548        if (err)
1549                goto put_b;
1550
1551        /* We use the already reserved extra page in context state */
1552        if (!a->wa_bb_page) {
1553                GEM_BUG_ON(b->wa_bb_page);
1554                GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1555                goto unpin_b;
1556        }
1557
1558        /*
1559         * In order to test that our per context bb is truly per context,
1560         * and executes at the intended spot on context restoring process,
1561         * make the batch store the ring start value to memory.
1562         * As ring start is restored apriori of starting the indirect ctx bb and
1563         * as it will be different for each context, it fits to this purpose.
1564         */
1565        indirect_ctx_bb_setup(a);
1566        indirect_ctx_bb_setup(b);
1567
1568        err = indirect_ctx_bb_check(a);
1569        if (err)
1570                goto unpin_b;
1571
1572        err = indirect_ctx_bb_check(b);
1573
1574unpin_b:
1575        intel_context_unpin(b);
1576put_b:
1577        intel_context_put(b);
1578unpin_a:
1579        intel_context_unpin(a);
1580put_a:
1581        intel_context_put(a);
1582
1583        return err;
1584}
1585
1586static int live_lrc_indirect_ctx_bb(void *arg)
1587{
1588        struct intel_gt *gt = arg;
1589        struct intel_engine_cs *engine;
1590        enum intel_engine_id id;
1591        int err = 0;
1592
1593        for_each_engine(engine, gt, id) {
1594                intel_engine_pm_get(engine);
1595                err = __live_lrc_indirect_ctx_bb(engine);
1596                intel_engine_pm_put(engine);
1597
1598                if (igt_flush_test(gt->i915))
1599                        err = -EIO;
1600
1601                if (err)
1602                        break;
1603        }
1604
1605        return err;
1606}
1607
1608static void garbage_reset(struct intel_engine_cs *engine,
1609                          struct i915_request *rq)
1610{
1611        const unsigned int bit = I915_RESET_ENGINE + engine->id;
1612        unsigned long *lock = &engine->gt->reset.flags;
1613
1614        local_bh_disable();
1615        if (!test_and_set_bit(bit, lock)) {
1616                tasklet_disable(&engine->execlists.tasklet);
1617
1618                if (!rq->fence.error)
1619                        __intel_engine_reset_bh(engine, NULL);
1620
1621                tasklet_enable(&engine->execlists.tasklet);
1622                clear_and_wake_up_bit(bit, lock);
1623        }
1624        local_bh_enable();
1625}
1626
1627static struct i915_request *garbage(struct intel_context *ce,
1628                                    struct rnd_state *prng)
1629{
1630        struct i915_request *rq;
1631        int err;
1632
1633        err = intel_context_pin(ce);
1634        if (err)
1635                return ERR_PTR(err);
1636
1637        prandom_bytes_state(prng,
1638                            ce->lrc_reg_state,
1639                            ce->engine->context_size -
1640                            LRC_STATE_OFFSET);
1641
1642        rq = intel_context_create_request(ce);
1643        if (IS_ERR(rq)) {
1644                err = PTR_ERR(rq);
1645                goto err_unpin;
1646        }
1647
1648        i915_request_get(rq);
1649        i915_request_add(rq);
1650        return rq;
1651
1652err_unpin:
1653        intel_context_unpin(ce);
1654        return ERR_PTR(err);
1655}
1656
1657static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1658{
1659        struct intel_context *ce;
1660        struct i915_request *hang;
1661        int err = 0;
1662
1663        ce = intel_context_create(engine);
1664        if (IS_ERR(ce))
1665                return PTR_ERR(ce);
1666
1667        hang = garbage(ce, prng);
1668        if (IS_ERR(hang)) {
1669                err = PTR_ERR(hang);
1670                goto err_ce;
1671        }
1672
1673        if (wait_for_submit(engine, hang, HZ / 2)) {
1674                i915_request_put(hang);
1675                err = -ETIME;
1676                goto err_ce;
1677        }
1678
1679        intel_context_set_banned(ce);
1680        garbage_reset(engine, hang);
1681
1682        intel_engine_flush_submission(engine);
1683        if (!hang->fence.error) {
1684                i915_request_put(hang);
1685                pr_err("%s: corrupted context was not reset\n",
1686                       engine->name);
1687                err = -EINVAL;
1688                goto err_ce;
1689        }
1690
1691        if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1692                pr_err("%s: corrupted context did not recover\n",
1693                       engine->name);
1694                i915_request_put(hang);
1695                err = -EIO;
1696                goto err_ce;
1697        }
1698        i915_request_put(hang);
1699
1700err_ce:
1701        intel_context_put(ce);
1702        return err;
1703}
1704
1705static int live_lrc_garbage(void *arg)
1706{
1707        struct intel_gt *gt = arg;
1708        struct intel_engine_cs *engine;
1709        enum intel_engine_id id;
1710
1711        /*
1712         * Verify that we can recover if one context state is completely
1713         * corrupted.
1714         */
1715
1716        if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1717                return 0;
1718
1719        for_each_engine(engine, gt, id) {
1720                I915_RND_STATE(prng);
1721                int err = 0, i;
1722
1723                if (!intel_has_reset_engine(engine->gt))
1724                        continue;
1725
1726                intel_engine_pm_get(engine);
1727                for (i = 0; i < 3; i++) {
1728                        err = __lrc_garbage(engine, &prng);
1729                        if (err)
1730                                break;
1731                }
1732                intel_engine_pm_put(engine);
1733
1734                if (igt_flush_test(gt->i915))
1735                        err = -EIO;
1736                if (err)
1737                        return err;
1738        }
1739
1740        return 0;
1741}
1742
1743static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1744{
1745        struct intel_context *ce;
1746        struct i915_request *rq;
1747        IGT_TIMEOUT(end_time);
1748        int err;
1749
1750        ce = intel_context_create(engine);
1751        if (IS_ERR(ce))
1752                return PTR_ERR(ce);
1753
1754        ce->runtime.num_underflow = 0;
1755        ce->runtime.max_underflow = 0;
1756
1757        do {
1758                unsigned int loop = 1024;
1759
1760                while (loop) {
1761                        rq = intel_context_create_request(ce);
1762                        if (IS_ERR(rq)) {
1763                                err = PTR_ERR(rq);
1764                                goto err_rq;
1765                        }
1766
1767                        if (--loop == 0)
1768                                i915_request_get(rq);
1769
1770                        i915_request_add(rq);
1771                }
1772
1773                if (__igt_timeout(end_time, NULL))
1774                        break;
1775
1776                i915_request_put(rq);
1777        } while (1);
1778
1779        err = i915_request_wait(rq, 0, HZ / 5);
1780        if (err < 0) {
1781                pr_err("%s: request not completed!\n", engine->name);
1782                goto err_wait;
1783        }
1784
1785        igt_flush_test(engine->i915);
1786
1787        pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1788                engine->name,
1789                intel_context_get_total_runtime_ns(ce),
1790                intel_context_get_avg_runtime_ns(ce));
1791
1792        err = 0;
1793        if (ce->runtime.num_underflow) {
1794                pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1795                       engine->name,
1796                       ce->runtime.num_underflow,
1797                       ce->runtime.max_underflow);
1798                GEM_TRACE_DUMP();
1799                err = -EOVERFLOW;
1800        }
1801
1802err_wait:
1803        i915_request_put(rq);
1804err_rq:
1805        intel_context_put(ce);
1806        return err;
1807}
1808
1809static int live_pphwsp_runtime(void *arg)
1810{
1811        struct intel_gt *gt = arg;
1812        struct intel_engine_cs *engine;
1813        enum intel_engine_id id;
1814        int err = 0;
1815
1816        /*
1817         * Check that cumulative context runtime as stored in the pphwsp[16]
1818         * is monotonic.
1819         */
1820
1821        for_each_engine(engine, gt, id) {
1822                err = __live_pphwsp_runtime(engine);
1823                if (err)
1824                        break;
1825        }
1826
1827        if (igt_flush_test(gt->i915))
1828                err = -EIO;
1829
1830        return err;
1831}
1832
1833int intel_lrc_live_selftests(struct drm_i915_private *i915)
1834{
1835        static const struct i915_subtest tests[] = {
1836                SUBTEST(live_lrc_layout),
1837                SUBTEST(live_lrc_fixed),
1838                SUBTEST(live_lrc_state),
1839                SUBTEST(live_lrc_gpr),
1840                SUBTEST(live_lrc_isolation),
1841                SUBTEST(live_lrc_timestamp),
1842                SUBTEST(live_lrc_garbage),
1843                SUBTEST(live_pphwsp_runtime),
1844                SUBTEST(live_lrc_indirect_ctx_bb),
1845        };
1846
1847        if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1848                return 0;
1849
1850        return intel_gt_live_subtests(tests, &i915->gt);
1851}
1852
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.