linux/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
<<
>>
Prefs
   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2016-2019 Intel Corporation
   4 */
   5
   6#include "i915_drv.h"
   7#include "intel_guc_ct.h"
   8#include "gt/intel_gt.h"
   9
  10static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
  11{
  12        return container_of(ct, struct intel_guc, ct);
  13}
  14
  15static inline struct intel_gt *ct_to_gt(struct intel_guc_ct *ct)
  16{
  17        return guc_to_gt(ct_to_guc(ct));
  18}
  19
  20static inline struct drm_i915_private *ct_to_i915(struct intel_guc_ct *ct)
  21{
  22        return ct_to_gt(ct)->i915;
  23}
  24
  25static inline struct drm_device *ct_to_drm(struct intel_guc_ct *ct)
  26{
  27        return &ct_to_i915(ct)->drm;
  28}
  29
  30#define CT_ERROR(_ct, _fmt, ...) \
  31        drm_err(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__)
  32#ifdef CONFIG_DRM_I915_DEBUG_GUC
  33#define CT_DEBUG(_ct, _fmt, ...) \
  34        drm_dbg(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__)
  35#else
  36#define CT_DEBUG(...)   do { } while (0)
  37#endif
  38#define CT_PROBE_ERROR(_ct, _fmt, ...) \
  39        i915_probe_error(ct_to_i915(ct), "CT: " _fmt, ##__VA_ARGS__)
  40
  41/**
  42 * DOC: CTB Blob
  43 *
  44 * We allocate single blob to hold both CTB descriptors and buffers:
  45 *
  46 *      +--------+-----------------------------------------------+------+
  47 *      | offset | contents                                      | size |
  48 *      +========+===============================================+======+
  49 *      | 0x0000 | H2G `CTB Descriptor`_ (send)                  |      |
  50 *      +--------+-----------------------------------------------+  4K  |
  51 *      | 0x0800 | G2H `CTB Descriptor`_ (recv)                  |      |
  52 *      +--------+-----------------------------------------------+------+
  53 *      | 0x1000 | H2G `CT Buffer`_ (send)                       | n*4K |
  54 *      |        |                                               |      |
  55 *      +--------+-----------------------------------------------+------+
  56 *      | 0x1000 | G2H `CT Buffer`_ (recv)                       | m*4K |
  57 *      | + n*4K |                                               |      |
  58 *      +--------+-----------------------------------------------+------+
  59 *
  60 * Size of each `CT Buffer`_ must be multiple of 4K.
  61 * As we don't expect too many messages, for now use minimum sizes.
  62 */
  63#define CTB_DESC_SIZE           ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
  64#define CTB_H2G_BUFFER_SIZE     (SZ_4K)
  65#define CTB_G2H_BUFFER_SIZE     (SZ_4K)
  66
  67struct ct_request {
  68        struct list_head link;
  69        u32 fence;
  70        u32 status;
  71        u32 response_len;
  72        u32 *response_buf;
  73};
  74
  75struct ct_incoming_msg {
  76        struct list_head link;
  77        u32 size;
  78        u32 msg[];
  79};
  80
  81enum { CTB_SEND = 0, CTB_RECV = 1 };
  82
  83enum { CTB_OWNER_HOST = 0 };
  84
  85static void ct_receive_tasklet_func(struct tasklet_struct *t);
  86static void ct_incoming_request_worker_func(struct work_struct *w);
  87
  88/**
  89 * intel_guc_ct_init_early - Initialize CT state without requiring device access
  90 * @ct: pointer to CT struct
  91 */
  92void intel_guc_ct_init_early(struct intel_guc_ct *ct)
  93{
  94        spin_lock_init(&ct->ctbs.send.lock);
  95        spin_lock_init(&ct->ctbs.recv.lock);
  96        spin_lock_init(&ct->requests.lock);
  97        INIT_LIST_HEAD(&ct->requests.pending);
  98        INIT_LIST_HEAD(&ct->requests.incoming);
  99        INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
 100        tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);
 101}
 102
 103static inline const char *guc_ct_buffer_type_to_str(u32 type)
 104{
 105        switch (type) {
 106        case INTEL_GUC_CT_BUFFER_TYPE_SEND:
 107                return "SEND";
 108        case INTEL_GUC_CT_BUFFER_TYPE_RECV:
 109                return "RECV";
 110        default:
 111                return "<invalid>";
 112        }
 113}
 114
 115static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc,
 116                                    u32 cmds_addr, u32 size)
 117{
 118        memset(desc, 0, sizeof(*desc));
 119        desc->addr = cmds_addr;
 120        desc->size = size;
 121        desc->owner = CTB_OWNER_HOST;
 122}
 123
 124static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb, u32 cmds_addr)
 125{
 126        guc_ct_buffer_desc_init(ctb->desc, cmds_addr, ctb->size);
 127}
 128
 129static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
 130                               struct guc_ct_buffer_desc *desc,
 131                               u32 *cmds, u32 size)
 132{
 133        GEM_BUG_ON(size % 4);
 134
 135        ctb->desc = desc;
 136        ctb->cmds = cmds;
 137        ctb->size = size;
 138
 139        guc_ct_buffer_reset(ctb, 0);
 140}
 141
 142static int guc_action_register_ct_buffer(struct intel_guc *guc,
 143                                         u32 desc_addr,
 144                                         u32 type)
 145{
 146        u32 action[] = {
 147                INTEL_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER,
 148                desc_addr,
 149                sizeof(struct guc_ct_buffer_desc),
 150                type
 151        };
 152
 153        /* Can't use generic send(), CT registration must go over MMIO */
 154        return intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0);
 155}
 156
 157static int ct_register_buffer(struct intel_guc_ct *ct, u32 desc_addr, u32 type)
 158{
 159        int err = guc_action_register_ct_buffer(ct_to_guc(ct), desc_addr, type);
 160
 161        if (unlikely(err))
 162                CT_ERROR(ct, "Failed to register %s buffer (err=%d)\n",
 163                         guc_ct_buffer_type_to_str(type), err);
 164        return err;
 165}
 166
 167static int guc_action_deregister_ct_buffer(struct intel_guc *guc, u32 type)
 168{
 169        u32 action[] = {
 170                INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER,
 171                CTB_OWNER_HOST,
 172                type
 173        };
 174
 175        /* Can't use generic send(), CT deregistration must go over MMIO */
 176        return intel_guc_send_mmio(guc, action, ARRAY_SIZE(action), NULL, 0);
 177}
 178
 179static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type)
 180{
 181        int err = guc_action_deregister_ct_buffer(ct_to_guc(ct), type);
 182
 183        if (unlikely(err))
 184                CT_ERROR(ct, "Failed to deregister %s buffer (err=%d)\n",
 185                         guc_ct_buffer_type_to_str(type), err);
 186        return err;
 187}
 188
 189/**
 190 * intel_guc_ct_init - Init buffer-based communication
 191 * @ct: pointer to CT struct
 192 *
 193 * Allocate memory required for buffer-based communication.
 194 *
 195 * Return: 0 on success, a negative errno code on failure.
 196 */
 197int intel_guc_ct_init(struct intel_guc_ct *ct)
 198{
 199        struct intel_guc *guc = ct_to_guc(ct);
 200        struct guc_ct_buffer_desc *desc;
 201        u32 blob_size;
 202        u32 cmds_size;
 203        void *blob;
 204        u32 *cmds;
 205        int err;
 206
 207        GEM_BUG_ON(ct->vma);
 208
 209        blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
 210        err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);
 211        if (unlikely(err)) {
 212                CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
 213                               blob_size, ERR_PTR(err));
 214                return err;
 215        }
 216
 217        CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
 218
 219        /* store pointers to desc and cmds for send ctb */
 220        desc = blob;
 221        cmds = blob + 2 * CTB_DESC_SIZE;
 222        cmds_size = CTB_H2G_BUFFER_SIZE;
 223        CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u\n", "send",
 224                 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size);
 225
 226        guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size);
 227
 228        /* store pointers to desc and cmds for recv ctb */
 229        desc = blob + CTB_DESC_SIZE;
 230        cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
 231        cmds_size = CTB_G2H_BUFFER_SIZE;
 232        CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u\n", "recv",
 233                 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size);
 234
 235        guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size);
 236
 237        return 0;
 238}
 239
 240/**
 241 * intel_guc_ct_fini - Fini buffer-based communication
 242 * @ct: pointer to CT struct
 243 *
 244 * Deallocate memory required for buffer-based communication.
 245 */
 246void intel_guc_ct_fini(struct intel_guc_ct *ct)
 247{
 248        GEM_BUG_ON(ct->enabled);
 249
 250        tasklet_kill(&ct->receive_tasklet);
 251        i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
 252        memset(ct, 0, sizeof(*ct));
 253}
 254
 255/**
 256 * intel_guc_ct_enable - Enable buffer based command transport.
 257 * @ct: pointer to CT struct
 258 *
 259 * Return: 0 on success, a negative errno code on failure.
 260 */
 261int intel_guc_ct_enable(struct intel_guc_ct *ct)
 262{
 263        struct intel_guc *guc = ct_to_guc(ct);
 264        u32 base, cmds;
 265        void *blob;
 266        int err;
 267
 268        GEM_BUG_ON(ct->enabled);
 269
 270        /* vma should be already allocated and map'ed */
 271        GEM_BUG_ON(!ct->vma);
 272        GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
 273        base = intel_guc_ggtt_offset(guc, ct->vma);
 274
 275        /* blob should start with send descriptor */
 276        blob = __px_vaddr(ct->vma->obj);
 277        GEM_BUG_ON(blob != ct->ctbs.send.desc);
 278
 279        /* (re)initialize descriptors */
 280        cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);
 281        guc_ct_buffer_reset(&ct->ctbs.send, cmds);
 282
 283        cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);
 284        guc_ct_buffer_reset(&ct->ctbs.recv, cmds);
 285
 286        /*
 287         * Register both CT buffers starting with RECV buffer.
 288         * Descriptors are in first half of the blob.
 289         */
 290        err = ct_register_buffer(ct, base + ptrdiff(ct->ctbs.recv.desc, blob),
 291                                 INTEL_GUC_CT_BUFFER_TYPE_RECV);
 292        if (unlikely(err))
 293                goto err_out;
 294
 295        err = ct_register_buffer(ct, base + ptrdiff(ct->ctbs.send.desc, blob),
 296                                 INTEL_GUC_CT_BUFFER_TYPE_SEND);
 297        if (unlikely(err))
 298                goto err_deregister;
 299
 300        ct->enabled = true;
 301
 302        return 0;
 303
 304err_deregister:
 305        ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_RECV);
 306err_out:
 307        CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
 308        return err;
 309}
 310
 311/**
 312 * intel_guc_ct_disable - Disable buffer based command transport.
 313 * @ct: pointer to CT struct
 314 */
 315void intel_guc_ct_disable(struct intel_guc_ct *ct)
 316{
 317        struct intel_guc *guc = ct_to_guc(ct);
 318
 319        GEM_BUG_ON(!ct->enabled);
 320
 321        ct->enabled = false;
 322
 323        if (intel_guc_is_fw_running(guc)) {
 324                ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_SEND);
 325                ct_deregister_buffer(ct, INTEL_GUC_CT_BUFFER_TYPE_RECV);
 326        }
 327}
 328
 329static u32 ct_get_next_fence(struct intel_guc_ct *ct)
 330{
 331        /* For now it's trivial */
 332        return ++ct->requests.last_fence;
 333}
 334
 335static void write_barrier(struct intel_guc_ct *ct)
 336{
 337        struct intel_guc *guc = ct_to_guc(ct);
 338        struct intel_gt *gt = guc_to_gt(guc);
 339
 340        if (i915_gem_object_is_lmem(guc->ct.vma->obj)) {
 341                GEM_BUG_ON(guc->send_regs.fw_domains);
 342                /*
 343                 * This register is used by the i915 and GuC for MMIO based
 344                 * communication. Once we are in this code CTBs are the only
 345                 * method the i915 uses to communicate with the GuC so it is
 346                 * safe to write to this register (a value of 0 is NOP for MMIO
 347                 * communication). If we ever start mixing CTBs and MMIOs a new
 348                 * register will have to be chosen.
 349                 */
 350                intel_uncore_write_fw(gt->uncore, GEN11_SOFT_SCRATCH(0), 0);
 351        } else {
 352                /* wmb() sufficient for a barrier if in smem */
 353                wmb();
 354        }
 355}
 356
 357/**
 358 * DOC: CTB Host to GuC request
 359 *
 360 * Format of the CTB Host to GuC request message is as follows::
 361 *
 362 *      +------------+---------+---------+---------+---------+
 363 *      |   msg[0]   |   [1]   |   [2]   |   ...   |  [n-1]  |
 364 *      +------------+---------+---------+---------+---------+
 365 *      |   MESSAGE  |       MESSAGE PAYLOAD                 |
 366 *      +   HEADER   +---------+---------+---------+---------+
 367 *      |            |    0    |    1    |   ...   |    n    |
 368 *      +============+=========+=========+=========+=========+
 369 *      |  len >= 1  |  FENCE  |     request specific data   |
 370 *      +------+-----+---------+---------+---------+---------+
 371 *
 372 *                   ^-----------------len-------------------^
 373 */
 374
 375static int ct_write(struct intel_guc_ct *ct,
 376                    const u32 *action,
 377                    u32 len /* in dwords */,
 378                    u32 fence)
 379{
 380        struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
 381        struct guc_ct_buffer_desc *desc = ctb->desc;
 382        u32 head = desc->head;
 383        u32 tail = desc->tail;
 384        u32 size = ctb->size;
 385        u32 used;
 386        u32 header;
 387        u32 *cmds = ctb->cmds;
 388        unsigned int i;
 389
 390        if (unlikely(desc->is_in_error))
 391                return -EPIPE;
 392
 393        if (unlikely(!IS_ALIGNED(head | tail, 4) ||
 394                     (tail | head) >= size))
 395                goto corrupted;
 396
 397        /* later calculations will be done in dwords */
 398        head /= 4;
 399        tail /= 4;
 400        size /= 4;
 401
 402        /*
 403         * tail == head condition indicates empty. GuC FW does not support
 404         * using up the entire buffer to get tail == head meaning full.
 405         */
 406        if (tail < head)
 407                used = (size - head) + tail;
 408        else
 409                used = tail - head;
 410
 411        /* make sure there is a space including extra dw for the fence */
 412        if (unlikely(used + len + 1 >= size))
 413                return -ENOSPC;
 414
 415        /*
 416         * Write the message. The format is the following:
 417         * DW0: header (including action code)
 418         * DW1: fence
 419         * DW2+: action data
 420         */
 421        header = (len << GUC_CT_MSG_LEN_SHIFT) |
 422                 GUC_CT_MSG_SEND_STATUS |
 423                 (action[0] << GUC_CT_MSG_ACTION_SHIFT);
 424
 425        CT_DEBUG(ct, "writing %*ph %*ph %*ph\n",
 426                 4, &header, 4, &fence, 4 * (len - 1), &action[1]);
 427
 428        cmds[tail] = header;
 429        tail = (tail + 1) % size;
 430
 431        cmds[tail] = fence;
 432        tail = (tail + 1) % size;
 433
 434        for (i = 1; i < len; i++) {
 435                cmds[tail] = action[i];
 436                tail = (tail + 1) % size;
 437        }
 438        GEM_BUG_ON(tail > size);
 439
 440        /*
 441         * make sure H2G buffer update and LRC tail update (if this triggering a
 442         * submission) are visible before updating the descriptor tail
 443         */
 444        write_barrier(ct);
 445
 446        /* now update desc tail (back in bytes) */
 447        desc->tail = tail * 4;
 448        return 0;
 449
 450corrupted:
 451        CT_ERROR(ct, "Corrupted descriptor addr=%#x head=%u tail=%u size=%u\n",
 452                 desc->addr, desc->head, desc->tail, desc->size);
 453        desc->is_in_error = 1;
 454        return -EPIPE;
 455}
 456
 457/**
 458 * wait_for_ct_request_update - Wait for CT request state update.
 459 * @req:        pointer to pending request
 460 * @status:     placeholder for status
 461 *
 462 * For each sent request, Guc shall send bac CT response message.
 463 * Our message handler will update status of tracked request once
 464 * response message with given fence is received. Wait here and
 465 * check for valid response status value.
 466 *
 467 * Return:
 468 * *    0 response received (status is valid)
 469 * *    -ETIMEDOUT no response within hardcoded timeout
 470 */
 471static int wait_for_ct_request_update(struct ct_request *req, u32 *status)
 472{
 473        int err;
 474
 475        /*
 476         * Fast commands should complete in less than 10us, so sample quickly
 477         * up to that length of time, then switch to a slower sleep-wait loop.
 478         * No GuC command should ever take longer than 10ms.
 479         */
 480#define done INTEL_GUC_MSG_IS_RESPONSE(READ_ONCE(req->status))
 481        err = wait_for_us(done, 10);
 482        if (err)
 483                err = wait_for(done, 10);
 484#undef done
 485
 486        if (unlikely(err))
 487                DRM_ERROR("CT: fence %u err %d\n", req->fence, err);
 488
 489        *status = req->status;
 490        return err;
 491}
 492
 493static int ct_send(struct intel_guc_ct *ct,
 494                   const u32 *action,
 495                   u32 len,
 496                   u32 *response_buf,
 497                   u32 response_buf_size,
 498                   u32 *status)
 499{
 500        struct ct_request request;
 501        unsigned long flags;
 502        u32 fence;
 503        int err;
 504
 505        GEM_BUG_ON(!ct->enabled);
 506        GEM_BUG_ON(!len);
 507        GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK);
 508        GEM_BUG_ON(!response_buf && response_buf_size);
 509
 510        spin_lock_irqsave(&ct->ctbs.send.lock, flags);
 511
 512        fence = ct_get_next_fence(ct);
 513        request.fence = fence;
 514        request.status = 0;
 515        request.response_len = response_buf_size;
 516        request.response_buf = response_buf;
 517
 518        spin_lock(&ct->requests.lock);
 519        list_add_tail(&request.link, &ct->requests.pending);
 520        spin_unlock(&ct->requests.lock);
 521
 522        err = ct_write(ct, action, len, fence);
 523
 524        spin_unlock_irqrestore(&ct->ctbs.send.lock, flags);
 525
 526        if (unlikely(err))
 527                goto unlink;
 528
 529        intel_guc_notify(ct_to_guc(ct));
 530
 531        err = wait_for_ct_request_update(&request, status);
 532        if (unlikely(err))
 533                goto unlink;
 534
 535        if (!INTEL_GUC_MSG_IS_RESPONSE_SUCCESS(*status)) {
 536                err = -EIO;
 537                goto unlink;
 538        }
 539
 540        if (response_buf) {
 541                /* There shall be no data in the status */
 542                WARN_ON(INTEL_GUC_MSG_TO_DATA(request.status));
 543                /* Return actual response len */
 544                err = request.response_len;
 545        } else {
 546                /* There shall be no response payload */
 547                WARN_ON(request.response_len);
 548                /* Return data decoded from the status dword */
 549                err = INTEL_GUC_MSG_TO_DATA(*status);
 550        }
 551
 552unlink:
 553        spin_lock_irqsave(&ct->requests.lock, flags);
 554        list_del(&request.link);
 555        spin_unlock_irqrestore(&ct->requests.lock, flags);
 556
 557        return err;
 558}
 559
 560/*
 561 * Command Transport (CT) buffer based GuC send function.
 562 */
 563int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
 564                      u32 *response_buf, u32 response_buf_size)
 565{
 566        u32 status = ~0; /* undefined */
 567        int ret;
 568
 569        if (unlikely(!ct->enabled)) {
 570                WARN(1, "Unexpected send: action=%#x\n", *action);
 571                return -ENODEV;
 572        }
 573
 574        ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
 575        if (unlikely(ret < 0)) {
 576                CT_ERROR(ct, "Sending action %#x failed (err=%d status=%#X)\n",
 577                         action[0], ret, status);
 578        } else if (unlikely(ret)) {
 579                CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
 580                         action[0], ret, ret);
 581        }
 582
 583        return ret;
 584}
 585
 586static inline unsigned int ct_header_get_len(u32 header)
 587{
 588        return (header >> GUC_CT_MSG_LEN_SHIFT) & GUC_CT_MSG_LEN_MASK;
 589}
 590
 591static inline unsigned int ct_header_get_action(u32 header)
 592{
 593        return (header >> GUC_CT_MSG_ACTION_SHIFT) & GUC_CT_MSG_ACTION_MASK;
 594}
 595
 596static inline bool ct_header_is_response(u32 header)
 597{
 598        return !!(header & GUC_CT_MSG_IS_RESPONSE);
 599}
 600
 601static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
 602{
 603        struct ct_incoming_msg *msg;
 604
 605        msg = kmalloc(sizeof(*msg) + sizeof(u32) * num_dwords, GFP_ATOMIC);
 606        if (msg)
 607                msg->size = num_dwords;
 608        return msg;
 609}
 610
 611static void ct_free_msg(struct ct_incoming_msg *msg)
 612{
 613        kfree(msg);
 614}
 615
 616/*
 617 * Return: number available remaining dwords to read (0 if empty)
 618 *         or a negative error code on failure
 619 */
 620static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
 621{
 622        struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
 623        struct guc_ct_buffer_desc *desc = ctb->desc;
 624        u32 head = desc->head;
 625        u32 tail = desc->tail;
 626        u32 size = ctb->size;
 627        u32 *cmds = ctb->cmds;
 628        s32 available;
 629        unsigned int len;
 630        unsigned int i;
 631        u32 header;
 632
 633        if (unlikely(desc->is_in_error))
 634                return -EPIPE;
 635
 636        if (unlikely(!IS_ALIGNED(head | tail, 4) ||
 637                     (tail | head) >= size))
 638                goto corrupted;
 639
 640        /* later calculations will be done in dwords */
 641        head /= 4;
 642        tail /= 4;
 643        size /= 4;
 644
 645        /* tail == head condition indicates empty */
 646        available = tail - head;
 647        if (unlikely(available == 0)) {
 648                *msg = NULL;
 649                return 0;
 650        }
 651
 652        /* beware of buffer wrap case */
 653        if (unlikely(available < 0))
 654                available += size;
 655        CT_DEBUG(ct, "available %d (%u:%u)\n", available, head, tail);
 656        GEM_BUG_ON(available < 0);
 657
 658        header = cmds[head];
 659        head = (head + 1) % size;
 660
 661        /* message len with header */
 662        len = ct_header_get_len(header) + 1;
 663        if (unlikely(len > (u32)available)) {
 664                CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
 665                         4, &header,
 666                         4 * (head + available - 1 > size ?
 667                              size - head : available - 1), &cmds[head],
 668                         4 * (head + available - 1 > size ?
 669                              available - 1 - size + head : 0), &cmds[0]);
 670                goto corrupted;
 671        }
 672
 673        *msg = ct_alloc_msg(len);
 674        if (!*msg) {
 675                CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
 676                         4, &header,
 677                         4 * (head + available - 1 > size ?
 678                              size - head : available - 1), &cmds[head],
 679                         4 * (head + available - 1 > size ?
 680                              available - 1 - size + head : 0), &cmds[0]);
 681                return available;
 682        }
 683
 684        (*msg)->msg[0] = header;
 685
 686        for (i = 1; i < len; i++) {
 687                (*msg)->msg[i] = cmds[head];
 688                head = (head + 1) % size;
 689        }
 690        CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
 691
 692        desc->head = head * 4;
 693        return available - len;
 694
 695corrupted:
 696        CT_ERROR(ct, "Corrupted descriptor addr=%#x head=%u tail=%u size=%u\n",
 697                 desc->addr, desc->head, desc->tail, desc->size);
 698        desc->is_in_error = 1;
 699        return -EPIPE;
 700}
 701
 702/**
 703 * DOC: CTB GuC to Host response
 704 *
 705 * Format of the CTB GuC to Host response message is as follows::
 706 *
 707 *      +------------+---------+---------+---------+---------+---------+
 708 *      |   msg[0]   |   [1]   |   [2]   |   [3]   |   ...   |  [n-1]  |
 709 *      +------------+---------+---------+---------+---------+---------+
 710 *      |   MESSAGE  |       MESSAGE PAYLOAD                           |
 711 *      +   HEADER   +---------+---------+---------+---------+---------+
 712 *      |            |    0    |    1    |    2    |   ...   |    n    |
 713 *      +============+=========+=========+=========+=========+=========+
 714 *      |  len >= 2  |  FENCE  |  STATUS |   response specific data    |
 715 *      +------+-----+---------+---------+---------+---------+---------+
 716 *
 717 *                   ^-----------------------len-----------------------^
 718 */
 719
 720static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
 721{
 722        u32 header = response->msg[0];
 723        u32 len = ct_header_get_len(header);
 724        u32 fence;
 725        u32 status;
 726        u32 datalen;
 727        struct ct_request *req;
 728        unsigned long flags;
 729        bool found = false;
 730        int err = 0;
 731
 732        GEM_BUG_ON(!ct_header_is_response(header));
 733
 734        /* Response payload shall at least include fence and status */
 735        if (unlikely(len < 2)) {
 736                CT_ERROR(ct, "Corrupted response (len %u)\n", len);
 737                return -EPROTO;
 738        }
 739
 740        fence = response->msg[1];
 741        status = response->msg[2];
 742        datalen = len - 2;
 743
 744        /* Format of the status follows RESPONSE message */
 745        if (unlikely(!INTEL_GUC_MSG_IS_RESPONSE(status))) {
 746                CT_ERROR(ct, "Corrupted response (status %#x)\n", status);
 747                return -EPROTO;
 748        }
 749
 750        CT_DEBUG(ct, "response fence %u status %#x\n", fence, status);
 751
 752        spin_lock_irqsave(&ct->requests.lock, flags);
 753        list_for_each_entry(req, &ct->requests.pending, link) {
 754                if (unlikely(fence != req->fence)) {
 755                        CT_DEBUG(ct, "request %u awaits response\n",
 756                                 req->fence);
 757                        continue;
 758                }
 759                if (unlikely(datalen > req->response_len)) {
 760                        CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
 761                                 req->fence, datalen, req->response_len);
 762                        datalen = min(datalen, req->response_len);
 763                        err = -EMSGSIZE;
 764                }
 765                if (datalen)
 766                        memcpy(req->response_buf, response->msg + 3, 4 * datalen);
 767                req->response_len = datalen;
 768                WRITE_ONCE(req->status, status);
 769                found = true;
 770                break;
 771        }
 772        spin_unlock_irqrestore(&ct->requests.lock, flags);
 773
 774        if (!found) {
 775                CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence);
 776                return -ENOKEY;
 777        }
 778
 779        if (unlikely(err))
 780                return err;
 781
 782        ct_free_msg(response);
 783        return 0;
 784}
 785
 786static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
 787{
 788        struct intel_guc *guc = ct_to_guc(ct);
 789        u32 header, action, len;
 790        const u32 *payload;
 791        int ret;
 792
 793        header = request->msg[0];
 794        payload = &request->msg[1];
 795        action = ct_header_get_action(header);
 796        len = ct_header_get_len(header);
 797
 798        CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
 799
 800        switch (action) {
 801        case INTEL_GUC_ACTION_DEFAULT:
 802                ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
 803                break;
 804        default:
 805                ret = -EOPNOTSUPP;
 806                break;
 807        }
 808
 809        if (unlikely(ret)) {
 810                CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
 811                         action, ERR_PTR(ret));
 812                return ret;
 813        }
 814
 815        ct_free_msg(request);
 816        return 0;
 817}
 818
 819static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
 820{
 821        unsigned long flags;
 822        struct ct_incoming_msg *request;
 823        bool done;
 824        int err;
 825
 826        spin_lock_irqsave(&ct->requests.lock, flags);
 827        request = list_first_entry_or_null(&ct->requests.incoming,
 828                                           struct ct_incoming_msg, link);
 829        if (request)
 830                list_del(&request->link);
 831        done = !!list_empty(&ct->requests.incoming);
 832        spin_unlock_irqrestore(&ct->requests.lock, flags);
 833
 834        if (!request)
 835                return true;
 836
 837        err = ct_process_request(ct, request);
 838        if (unlikely(err)) {
 839                CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
 840                         ERR_PTR(err), 4 * request->size, request->msg);
 841                ct_free_msg(request);
 842        }
 843
 844        return done;
 845}
 846
 847static void ct_incoming_request_worker_func(struct work_struct *w)
 848{
 849        struct intel_guc_ct *ct =
 850                container_of(w, struct intel_guc_ct, requests.worker);
 851        bool done;
 852
 853        done = ct_process_incoming_requests(ct);
 854        if (!done)
 855                queue_work(system_unbound_wq, &ct->requests.worker);
 856}
 857
 858/**
 859 * DOC: CTB GuC to Host request
 860 *
 861 * Format of the CTB GuC to Host request message is as follows::
 862 *
 863 *      +------------+---------+---------+---------+---------+---------+
 864 *      |   msg[0]   |   [1]   |   [2]   |   [3]   |   ...   |  [n-1]  |
 865 *      +------------+---------+---------+---------+---------+---------+
 866 *      |   MESSAGE  |       MESSAGE PAYLOAD                           |
 867 *      +   HEADER   +---------+---------+---------+---------+---------+
 868 *      |            |    0    |    1    |    2    |   ...   |    n    |
 869 *      +============+=========+=========+=========+=========+=========+
 870 *      |     len    |            request specific data                |
 871 *      +------+-----+---------+---------+---------+---------+---------+
 872 *
 873 *                   ^-----------------------len-----------------------^
 874 */
 875
 876static int ct_handle_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
 877{
 878        unsigned long flags;
 879
 880        GEM_BUG_ON(ct_header_is_response(request->msg[0]));
 881
 882        spin_lock_irqsave(&ct->requests.lock, flags);
 883        list_add_tail(&request->link, &ct->requests.incoming);
 884        spin_unlock_irqrestore(&ct->requests.lock, flags);
 885
 886        queue_work(system_unbound_wq, &ct->requests.worker);
 887        return 0;
 888}
 889
 890static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
 891{
 892        u32 header = msg->msg[0];
 893        int err;
 894
 895        if (ct_header_is_response(header))
 896                err = ct_handle_response(ct, msg);
 897        else
 898                err = ct_handle_request(ct, msg);
 899
 900        if (unlikely(err)) {
 901                CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
 902                         ERR_PTR(err), 4 * msg->size, msg->msg);
 903                ct_free_msg(msg);
 904        }
 905}
 906
 907/*
 908 * Return: number available remaining dwords to read (0 if empty)
 909 *         or a negative error code on failure
 910 */
 911static int ct_receive(struct intel_guc_ct *ct)
 912{
 913        struct ct_incoming_msg *msg = NULL;
 914        unsigned long flags;
 915        int ret;
 916
 917        spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
 918        ret = ct_read(ct, &msg);
 919        spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);
 920        if (ret < 0)
 921                return ret;
 922
 923        if (msg)
 924                ct_handle_msg(ct, msg);
 925
 926        return ret;
 927}
 928
 929static void ct_try_receive_message(struct intel_guc_ct *ct)
 930{
 931        int ret;
 932
 933        if (GEM_WARN_ON(!ct->enabled))
 934                return;
 935
 936        ret = ct_receive(ct);
 937        if (ret > 0)
 938                tasklet_hi_schedule(&ct->receive_tasklet);
 939}
 940
 941static void ct_receive_tasklet_func(struct tasklet_struct *t)
 942{
 943        struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
 944
 945        ct_try_receive_message(ct);
 946}
 947
 948/*
 949 * When we're communicating with the GuC over CT, GuC uses events
 950 * to notify us about new messages being posted on the RECV buffer.
 951 */
 952void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
 953{
 954        if (unlikely(!ct->enabled)) {
 955                WARN(1, "Unexpected GuC event received while CT disabled!\n");
 956                return;
 957        }
 958
 959        ct_try_receive_message(ct);
 960}
 961
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.