linux/drivers/scsi/cxlflash/superpipe.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * CXL Flash Device Driver
   4 *
   5 * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
   6 *             Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
   7 *
   8 * Copyright (C) 2015 IBM Corporation
   9 */
  10
  11#include <linux/delay.h>
  12#include <linux/file.h>
  13#include <linux/interrupt.h>
  14#include <linux/pci.h>
  15#include <linux/syscalls.h>
  16#include <asm/unaligned.h>
  17
  18#include <scsi/scsi.h>
  19#include <scsi/scsi_host.h>
  20#include <scsi/scsi_cmnd.h>
  21#include <scsi/scsi_eh.h>
  22#include <uapi/scsi/cxlflash_ioctl.h>
  23
  24#include "sislite.h"
  25#include "common.h"
  26#include "vlun.h"
  27#include "superpipe.h"
  28
  29struct cxlflash_global global;
  30
  31/**
  32 * marshal_rele_to_resize() - translate release to resize structure
  33 * @release:    Source structure from which to translate/copy.
  34 * @resize:     Destination structure for the translate/copy.
  35 */
  36static void marshal_rele_to_resize(struct dk_cxlflash_release *release,
  37                                   struct dk_cxlflash_resize *resize)
  38{
  39        resize->hdr = release->hdr;
  40        resize->context_id = release->context_id;
  41        resize->rsrc_handle = release->rsrc_handle;
  42}
  43
  44/**
  45 * marshal_det_to_rele() - translate detach to release structure
  46 * @detach:     Destination structure for the translate/copy.
  47 * @release:    Source structure from which to translate/copy.
  48 */
  49static void marshal_det_to_rele(struct dk_cxlflash_detach *detach,
  50                                struct dk_cxlflash_release *release)
  51{
  52        release->hdr = detach->hdr;
  53        release->context_id = detach->context_id;
  54}
  55
  56/**
  57 * marshal_udir_to_rele() - translate udirect to release structure
  58 * @udirect:    Source structure from which to translate/copy.
  59 * @release:    Destination structure for the translate/copy.
  60 */
  61static void marshal_udir_to_rele(struct dk_cxlflash_udirect *udirect,
  62                                 struct dk_cxlflash_release *release)
  63{
  64        release->hdr = udirect->hdr;
  65        release->context_id = udirect->context_id;
  66        release->rsrc_handle = udirect->rsrc_handle;
  67}
  68
  69/**
  70 * cxlflash_free_errpage() - frees resources associated with global error page
  71 */
  72void cxlflash_free_errpage(void)
  73{
  74
  75        mutex_lock(&global.mutex);
  76        if (global.err_page) {
  77                __free_page(global.err_page);
  78                global.err_page = NULL;
  79        }
  80        mutex_unlock(&global.mutex);
  81}
  82
  83/**
  84 * cxlflash_stop_term_user_contexts() - stops/terminates known user contexts
  85 * @cfg:        Internal structure associated with the host.
  86 *
  87 * When the host needs to go down, all users must be quiesced and their
  88 * memory freed. This is accomplished by putting the contexts in error
  89 * state which will notify the user and let them 'drive' the tear down.
  90 * Meanwhile, this routine camps until all user contexts have been removed.
  91 *
  92 * Note that the main loop in this routine will always execute at least once
  93 * to flush the reset_waitq.
  94 */
  95void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg)
  96{
  97        struct device *dev = &cfg->dev->dev;
  98        int i, found = true;
  99
 100        cxlflash_mark_contexts_error(cfg);
 101
 102        while (true) {
 103                for (i = 0; i < MAX_CONTEXT; i++)
 104                        if (cfg->ctx_tbl[i]) {
 105                                found = true;
 106                                break;
 107                        }
 108
 109                if (!found && list_empty(&cfg->ctx_err_recovery))
 110                        return;
 111
 112                dev_dbg(dev, "%s: Wait for user contexts to quiesce...\n",
 113                        __func__);
 114                wake_up_all(&cfg->reset_waitq);
 115                ssleep(1);
 116                found = false;
 117        }
 118}
 119
 120/**
 121 * find_error_context() - locates a context by cookie on the error recovery list
 122 * @cfg:        Internal structure associated with the host.
 123 * @rctxid:     Desired context by id.
 124 * @file:       Desired context by file.
 125 *
 126 * Return: Found context on success, NULL on failure
 127 */
 128static struct ctx_info *find_error_context(struct cxlflash_cfg *cfg, u64 rctxid,
 129                                           struct file *file)
 130{
 131        struct ctx_info *ctxi;
 132
 133        list_for_each_entry(ctxi, &cfg->ctx_err_recovery, list)
 134                if ((ctxi->ctxid == rctxid) || (ctxi->file == file))
 135                        return ctxi;
 136
 137        return NULL;
 138}
 139
 140/**
 141 * get_context() - obtains a validated and locked context reference
 142 * @cfg:        Internal structure associated with the host.
 143 * @rctxid:     Desired context (raw, un-decoded format).
 144 * @arg:        LUN information or file associated with request.
 145 * @ctx_ctrl:   Control information to 'steer' desired lookup.
 146 *
 147 * NOTE: despite the name pid, in linux, current->pid actually refers
 148 * to the lightweight process id (tid) and can change if the process is
 149 * multi threaded. The tgid remains constant for the process and only changes
 150 * when the process of fork. For all intents and purposes, think of tgid
 151 * as a pid in the traditional sense.
 152 *
 153 * Return: Validated context on success, NULL on failure
 154 */
 155struct ctx_info *get_context(struct cxlflash_cfg *cfg, u64 rctxid,
 156                             void *arg, enum ctx_ctrl ctx_ctrl)
 157{
 158        struct device *dev = &cfg->dev->dev;
 159        struct ctx_info *ctxi = NULL;
 160        struct lun_access *lun_access = NULL;
 161        struct file *file = NULL;
 162        struct llun_info *lli = arg;
 163        u64 ctxid = DECODE_CTXID(rctxid);
 164        int rc;
 165        pid_t pid = task_tgid_nr(current), ctxpid = 0;
 166
 167        if (ctx_ctrl & CTX_CTRL_FILE) {
 168                lli = NULL;
 169                file = (struct file *)arg;
 170        }
 171
 172        if (ctx_ctrl & CTX_CTRL_CLONE)
 173                pid = task_ppid_nr(current);
 174
 175        if (likely(ctxid < MAX_CONTEXT)) {
 176                while (true) {
 177                        mutex_lock(&cfg->ctx_tbl_list_mutex);
 178                        ctxi = cfg->ctx_tbl[ctxid];
 179                        if (ctxi)
 180                                if ((file && (ctxi->file != file)) ||
 181                                    (!file && (ctxi->ctxid != rctxid)))
 182                                        ctxi = NULL;
 183
 184                        if ((ctx_ctrl & CTX_CTRL_ERR) ||
 185                            (!ctxi && (ctx_ctrl & CTX_CTRL_ERR_FALLBACK)))
 186                                ctxi = find_error_context(cfg, rctxid, file);
 187                        if (!ctxi) {
 188                                mutex_unlock(&cfg->ctx_tbl_list_mutex);
 189                                goto out;
 190                        }
 191
 192                        /*
 193                         * Need to acquire ownership of the context while still
 194                         * under the table/list lock to serialize with a remove
 195                         * thread. Use the 'try' to avoid stalling the
 196                         * table/list lock for a single context.
 197                         *
 198                         * Note that the lock order is:
 199                         *
 200                         *      cfg->ctx_tbl_list_mutex -> ctxi->mutex
 201                         *
 202                         * Therefore release ctx_tbl_list_mutex before retrying.
 203                         */
 204                        rc = mutex_trylock(&ctxi->mutex);
 205                        mutex_unlock(&cfg->ctx_tbl_list_mutex);
 206                        if (rc)
 207                                break; /* got the context's lock! */
 208                }
 209
 210                if (ctxi->unavail)
 211                        goto denied;
 212
 213                ctxpid = ctxi->pid;
 214                if (likely(!(ctx_ctrl & CTX_CTRL_NOPID)))
 215                        if (pid != ctxpid)
 216                                goto denied;
 217
 218                if (lli) {
 219                        list_for_each_entry(lun_access, &ctxi->luns, list)
 220                                if (lun_access->lli == lli)
 221                                        goto out;
 222                        goto denied;
 223                }
 224        }
 225
 226out:
 227        dev_dbg(dev, "%s: rctxid=%016llx ctxinfo=%p ctxpid=%u pid=%u "
 228                "ctx_ctrl=%u\n", __func__, rctxid, ctxi, ctxpid, pid,
 229                ctx_ctrl);
 230
 231        return ctxi;
 232
 233denied:
 234        mutex_unlock(&ctxi->mutex);
 235        ctxi = NULL;
 236        goto out;
 237}
 238
 239/**
 240 * put_context() - release a context that was retrieved from get_context()
 241 * @ctxi:       Context to release.
 242 *
 243 * For now, releasing the context equates to unlocking it's mutex.
 244 */
 245void put_context(struct ctx_info *ctxi)
 246{
 247        mutex_unlock(&ctxi->mutex);
 248}
 249
 250/**
 251 * afu_attach() - attach a context to the AFU
 252 * @cfg:        Internal structure associated with the host.
 253 * @ctxi:       Context to attach.
 254 *
 255 * Upon setting the context capabilities, they must be confirmed with
 256 * a read back operation as the context might have been closed since
 257 * the mailbox was unlocked. When this occurs, registration is failed.
 258 *
 259 * Return: 0 on success, -errno on failure
 260 */
 261static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
 262{
 263        struct device *dev = &cfg->dev->dev;
 264        struct afu *afu = cfg->afu;
 265        struct sisl_ctrl_map __iomem *ctrl_map = ctxi->ctrl_map;
 266        int rc = 0;
 267        struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
 268        u64 val;
 269        int i;
 270
 271        /* Unlock cap and restrict user to read/write cmds in translated mode */
 272        readq_be(&ctrl_map->mbox_r);
 273        val = (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD);
 274        writeq_be(val, &ctrl_map->ctx_cap);
 275        val = readq_be(&ctrl_map->ctx_cap);
 276        if (val != (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD)) {
 277                dev_err(dev, "%s: ctx may be closed val=%016llx\n",
 278                        __func__, val);
 279                rc = -EAGAIN;
 280                goto out;
 281        }
 282
 283        if (afu_is_ocxl_lisn(afu)) {
 284                /* Set up the LISN effective address for each interrupt */
 285                for (i = 0; i < ctxi->irqs; i++) {
 286                        val = cfg->ops->get_irq_objhndl(ctxi->ctx, i);
 287                        writeq_be(val, &ctrl_map->lisn_ea[i]);
 288                }
 289
 290                /* Use primary HWQ PASID as identifier for all interrupts */
 291                val = hwq->ctx_hndl;
 292                writeq_be(SISL_LISN_PASID(val, val), &ctrl_map->lisn_pasid[0]);
 293                writeq_be(SISL_LISN_PASID(0UL, val), &ctrl_map->lisn_pasid[1]);
 294        }
 295
 296        /* Set up MMIO registers pointing to the RHT */
 297        writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);
 298        val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));
 299        writeq_be(val, &ctrl_map->rht_cnt_id);
 300out:
 301        dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
 302        return rc;
 303}
 304
 305/**
 306 * read_cap16() - issues a SCSI READ_CAP16 command
 307 * @sdev:       SCSI device associated with LUN.
 308 * @lli:        LUN destined for capacity request.
 309 *
 310 * The READ_CAP16 can take quite a while to complete. Should an EEH occur while
 311 * in scsi_execute(), the EEH handler will attempt to recover. As part of the
 312 * recovery, the handler drains all currently running ioctls, waiting until they
 313 * have completed before proceeding with a reset. As this routine is used on the
 314 * ioctl path, this can create a condition where the EEH handler becomes stuck,
 315 * infinitely waiting for this ioctl thread. To avoid this behavior, temporarily
 316 * unmark this thread as an ioctl thread by releasing the ioctl read semaphore.
 317 * This will allow the EEH handler to proceed with a recovery while this thread
 318 * is still running. Once the scsi_execute() returns, reacquire the ioctl read
 319 * semaphore and check the adapter state in case it changed while inside of
 320 * scsi_execute(). The state check will wait if the adapter is still being
 321 * recovered or return a failure if the recovery failed. In the event that the
 322 * adapter reset failed, simply return the failure as the ioctl would be unable
 323 * to continue.
 324 *
 325 * Note that the above puts a requirement on this routine to only be called on
 326 * an ioctl thread.
 327 *
 328 * Return: 0 on success, -errno on failure
 329 */
 330static int read_cap16(struct scsi_device *sdev, struct llun_info *lli)
 331{
 332        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
 333        struct device *dev = &cfg->dev->dev;
 334        struct glun_info *gli = lli->parent;
 335        struct scsi_sense_hdr sshdr;
 336        u8 *cmd_buf = NULL;
 337        u8 *scsi_cmd = NULL;
 338        int rc = 0;
 339        int result = 0;
 340        int retry_cnt = 0;
 341        u32 to = CMD_TIMEOUT * HZ;
 342
 343retry:
 344        cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);
 345        scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);
 346        if (unlikely(!cmd_buf || !scsi_cmd)) {
 347                rc = -ENOMEM;
 348                goto out;
 349        }
 350
 351        scsi_cmd[0] = SERVICE_ACTION_IN_16;     /* read cap(16) */
 352        scsi_cmd[1] = SAI_READ_CAPACITY_16;     /* service action */
 353        put_unaligned_be32(CMD_BUFSIZE, &scsi_cmd[10]);
 354
 355        dev_dbg(dev, "%s: %ssending cmd(%02x)\n", __func__,
 356                retry_cnt ? "re" : "", scsi_cmd[0]);
 357
 358        /* Drop the ioctl read semahpore across lengthy call */
 359        up_read(&cfg->ioctl_rwsem);
 360        result = scsi_execute(sdev, scsi_cmd, DMA_FROM_DEVICE, cmd_buf,
 361                              CMD_BUFSIZE, NULL, &sshdr, to, CMD_RETRIES,
 362                              0, 0, NULL);
 363        down_read(&cfg->ioctl_rwsem);
 364        rc = check_state(cfg);
 365        if (rc) {
 366                dev_err(dev, "%s: Failed state result=%08x\n",
 367                        __func__, result);
 368                rc = -ENODEV;
 369                goto out;
 370        }
 371
 372        if (driver_byte(result) == DRIVER_SENSE) {
 373                result &= ~(0xFF<<24); /* DRIVER_SENSE is not an error */
 374                if (result & SAM_STAT_CHECK_CONDITION) {
 375                        switch (sshdr.sense_key) {
 376                        case NO_SENSE:
 377                        case RECOVERED_ERROR:
 378                        case NOT_READY:
 379                                result &= ~SAM_STAT_CHECK_CONDITION;
 380                                break;
 381                        case UNIT_ATTENTION:
 382                                switch (sshdr.asc) {
 383                                case 0x29: /* Power on Reset or Device Reset */
 384                                        fallthrough;
 385                                case 0x2A: /* Device capacity changed */
 386                                case 0x3F: /* Report LUNs changed */
 387                                        /* Retry the command once more */
 388                                        if (retry_cnt++ < 1) {
 389                                                kfree(cmd_buf);
 390                                                kfree(scsi_cmd);
 391                                                goto retry;
 392                                        }
 393                                }
 394                                break;
 395                        default:
 396                                break;
 397                        }
 398                }
 399        }
 400
 401        if (result) {
 402                dev_err(dev, "%s: command failed, result=%08x\n",
 403                        __func__, result);
 404                rc = -EIO;
 405                goto out;
 406        }
 407
 408        /*
 409         * Read cap was successful, grab values from the buffer;
 410         * note that we don't need to worry about unaligned access
 411         * as the buffer is allocated on an aligned boundary.
 412         */
 413        mutex_lock(&gli->mutex);
 414        gli->max_lba = be64_to_cpu(*((__be64 *)&cmd_buf[0]));
 415        gli->blk_len = be32_to_cpu(*((__be32 *)&cmd_buf[8]));
 416        mutex_unlock(&gli->mutex);
 417
 418out:
 419        kfree(cmd_buf);
 420        kfree(scsi_cmd);
 421
 422        dev_dbg(dev, "%s: maxlba=%lld blklen=%d rc=%d\n",
 423                __func__, gli->max_lba, gli->blk_len, rc);
 424        return rc;
 425}
 426
 427/**
 428 * get_rhte() - obtains validated resource handle table entry reference
 429 * @ctxi:       Context owning the resource handle.
 430 * @rhndl:      Resource handle associated with entry.
 431 * @lli:        LUN associated with request.
 432 *
 433 * Return: Validated RHTE on success, NULL on failure
 434 */
 435struct sisl_rht_entry *get_rhte(struct ctx_info *ctxi, res_hndl_t rhndl,
 436                                struct llun_info *lli)
 437{
 438        struct cxlflash_cfg *cfg = ctxi->cfg;
 439        struct device *dev = &cfg->dev->dev;
 440        struct sisl_rht_entry *rhte = NULL;
 441
 442        if (unlikely(!ctxi->rht_start)) {
 443                dev_dbg(dev, "%s: Context does not have allocated RHT\n",
 444                         __func__);
 445                goto out;
 446        }
 447
 448        if (unlikely(rhndl >= MAX_RHT_PER_CONTEXT)) {
 449                dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
 450                        __func__, rhndl);
 451                goto out;
 452        }
 453
 454        if (unlikely(ctxi->rht_lun[rhndl] != lli)) {
 455                dev_dbg(dev, "%s: Bad resource handle LUN rhndl=%d\n",
 456                        __func__, rhndl);
 457                goto out;
 458        }
 459
 460        rhte = &ctxi->rht_start[rhndl];
 461        if (unlikely(rhte->nmask == 0)) {
 462                dev_dbg(dev, "%s: Unopened resource handle rhndl=%d\n",
 463                        __func__, rhndl);
 464                rhte = NULL;
 465                goto out;
 466        }
 467
 468out:
 469        return rhte;
 470}
 471
 472/**
 473 * rhte_checkout() - obtains free/empty resource handle table entry
 474 * @ctxi:       Context owning the resource handle.
 475 * @lli:        LUN associated with request.
 476 *
 477 * Return: Free RHTE on success, NULL on failure
 478 */
 479struct sisl_rht_entry *rhte_checkout(struct ctx_info *ctxi,
 480                                     struct llun_info *lli)
 481{
 482        struct cxlflash_cfg *cfg = ctxi->cfg;
 483        struct device *dev = &cfg->dev->dev;
 484        struct sisl_rht_entry *rhte = NULL;
 485        int i;
 486
 487        /* Find a free RHT entry */
 488        for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)
 489                if (ctxi->rht_start[i].nmask == 0) {
 490                        rhte = &ctxi->rht_start[i];
 491                        ctxi->rht_out++;
 492                        break;
 493                }
 494
 495        if (likely(rhte))
 496                ctxi->rht_lun[i] = lli;
 497
 498        dev_dbg(dev, "%s: returning rhte=%p index=%d\n", __func__, rhte, i);
 499        return rhte;
 500}
 501
 502/**
 503 * rhte_checkin() - releases a resource handle table entry
 504 * @ctxi:       Context owning the resource handle.
 505 * @rhte:       RHTE to release.
 506 */
 507void rhte_checkin(struct ctx_info *ctxi,
 508                  struct sisl_rht_entry *rhte)
 509{
 510        u32 rsrc_handle = rhte - ctxi->rht_start;
 511
 512        rhte->nmask = 0;
 513        rhte->fp = 0;
 514        ctxi->rht_out--;
 515        ctxi->rht_lun[rsrc_handle] = NULL;
 516        ctxi->rht_needs_ws[rsrc_handle] = false;
 517}
 518
 519/**
 520 * rht_format1() - populates a RHTE for format 1
 521 * @rhte:       RHTE to populate.
 522 * @lun_id:     LUN ID of LUN associated with RHTE.
 523 * @perm:       Desired permissions for RHTE.
 524 * @port_sel:   Port selection mask
 525 */
 526static void rht_format1(struct sisl_rht_entry *rhte, u64 lun_id, u32 perm,
 527                        u32 port_sel)
 528{
 529        /*
 530         * Populate the Format 1 RHT entry for direct access (physical
 531         * LUN) using the synchronization sequence defined in the
 532         * SISLite specification.
 533         */
 534        struct sisl_rht_entry_f1 dummy = { 0 };
 535        struct sisl_rht_entry_f1 *rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
 536
 537        memset(rhte_f1, 0, sizeof(*rhte_f1));
 538        rhte_f1->fp = SISL_RHT_FP(1U, 0);
 539        dma_wmb(); /* Make setting of format bit visible */
 540
 541        rhte_f1->lun_id = lun_id;
 542        dma_wmb(); /* Make setting of LUN id visible */
 543
 544        /*
 545         * Use a dummy RHT Format 1 entry to build the second dword
 546         * of the entry that must be populated in a single write when
 547         * enabled (valid bit set to TRUE).
 548         */
 549        dummy.valid = 0x80;
 550        dummy.fp = SISL_RHT_FP(1U, perm);
 551        dummy.port_sel = port_sel;
 552        rhte_f1->dw = dummy.dw;
 553
 554        dma_wmb(); /* Make remaining RHT entry fields visible */
 555}
 556
 557/**
 558 * cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode
 559 * @gli:        LUN to attach.
 560 * @mode:       Desired mode of the LUN.
 561 * @locked:     Mutex status on current thread.
 562 *
 563 * Return: 0 on success, -errno on failure
 564 */
 565int cxlflash_lun_attach(struct glun_info *gli, enum lun_mode mode, bool locked)
 566{
 567        int rc = 0;
 568
 569        if (!locked)
 570                mutex_lock(&gli->mutex);
 571
 572        if (gli->mode == MODE_NONE)
 573                gli->mode = mode;
 574        else if (gli->mode != mode) {
 575                pr_debug("%s: gli_mode=%d requested_mode=%d\n",
 576                         __func__, gli->mode, mode);
 577                rc = -EINVAL;
 578                goto out;
 579        }
 580
 581        gli->users++;
 582        WARN_ON(gli->users <= 0);
 583out:
 584        pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n",
 585                 __func__, rc, gli->mode, gli->users);
 586        if (!locked)
 587                mutex_unlock(&gli->mutex);
 588        return rc;
 589}
 590
 591/**
 592 * cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode
 593 * @gli:        LUN to detach.
 594 *
 595 * When resetting the mode, terminate block allocation resources as they
 596 * are no longer required (service is safe to call even when block allocation
 597 * resources were not present - such as when transitioning from physical mode).
 598 * These resources will be reallocated when needed (subsequent transition to
 599 * virtual mode).
 600 */
 601void cxlflash_lun_detach(struct glun_info *gli)
 602{
 603        mutex_lock(&gli->mutex);
 604        WARN_ON(gli->mode == MODE_NONE);
 605        if (--gli->users == 0) {
 606                gli->mode = MODE_NONE;
 607                cxlflash_ba_terminate(&gli->blka.ba_lun);
 608        }
 609        pr_debug("%s: gli->users=%u\n", __func__, gli->users);
 610        WARN_ON(gli->users < 0);
 611        mutex_unlock(&gli->mutex);
 612}
 613
 614/**
 615 * _cxlflash_disk_release() - releases the specified resource entry
 616 * @sdev:       SCSI device associated with LUN.
 617 * @ctxi:       Context owning resources.
 618 * @release:    Release ioctl data structure.
 619 *
 620 * For LUNs in virtual mode, the virtual LUN associated with the specified
 621 * resource handle is resized to 0 prior to releasing the RHTE. Note that the
 622 * AFU sync should _not_ be performed when the context is sitting on the error
 623 * recovery list. A context on the error recovery list is not known to the AFU
 624 * due to reset. When the context is recovered, it will be reattached and made
 625 * known again to the AFU.
 626 *
 627 * Return: 0 on success, -errno on failure
 628 */
 629int _cxlflash_disk_release(struct scsi_device *sdev,
 630                           struct ctx_info *ctxi,
 631                           struct dk_cxlflash_release *release)
 632{
 633        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
 634        struct device *dev = &cfg->dev->dev;
 635        struct llun_info *lli = sdev->hostdata;
 636        struct glun_info *gli = lli->parent;
 637        struct afu *afu = cfg->afu;
 638        bool put_ctx = false;
 639
 640        struct dk_cxlflash_resize size;
 641        res_hndl_t rhndl = release->rsrc_handle;
 642
 643        int rc = 0;
 644        int rcr = 0;
 645        u64 ctxid = DECODE_CTXID(release->context_id),
 646            rctxid = release->context_id;
 647
 648        struct sisl_rht_entry *rhte;
 649        struct sisl_rht_entry_f1 *rhte_f1;
 650
 651        dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu gli->mode=%u gli->users=%u\n",
 652                __func__, ctxid, release->rsrc_handle, gli->mode, gli->users);
 653
 654        if (!ctxi) {
 655                ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
 656                if (unlikely(!ctxi)) {
 657                        dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
 658                                __func__, ctxid);
 659                        rc = -EINVAL;
 660                        goto out;
 661                }
 662
 663                put_ctx = true;
 664        }
 665
 666        rhte = get_rhte(ctxi, rhndl, lli);
 667        if (unlikely(!rhte)) {
 668                dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
 669                        __func__, rhndl);
 670                rc = -EINVAL;
 671                goto out;
 672        }
 673
 674        /*
 675         * Resize to 0 for virtual LUNS by setting the size
 676         * to 0. This will clear LXT_START and LXT_CNT fields
 677         * in the RHT entry and properly sync with the AFU.
 678         *
 679         * Afterwards we clear the remaining fields.
 680         */
 681        switch (gli->mode) {
 682        case MODE_VIRTUAL:
 683                marshal_rele_to_resize(release, &size);
 684                size.req_size = 0;
 685                rc = _cxlflash_vlun_resize(sdev, ctxi, &size);
 686                if (rc) {
 687                        dev_dbg(dev, "%s: resize failed rc %d\n", __func__, rc);
 688                        goto out;
 689                }
 690
 691                break;
 692        case MODE_PHYSICAL:
 693                /*
 694                 * Clear the Format 1 RHT entry for direct access
 695                 * (physical LUN) using the synchronization sequence
 696                 * defined in the SISLite specification.
 697                 */
 698                rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
 699
 700                rhte_f1->valid = 0;
 701                dma_wmb(); /* Make revocation of RHT entry visible */
 702
 703                rhte_f1->lun_id = 0;
 704                dma_wmb(); /* Make clearing of LUN id visible */
 705
 706                rhte_f1->dw = 0;
 707                dma_wmb(); /* Make RHT entry bottom-half clearing visible */
 708
 709                if (!ctxi->err_recovery_active) {
 710                        rcr = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);
 711                        if (unlikely(rcr))
 712                                dev_dbg(dev, "%s: AFU sync failed rc=%d\n",
 713                                        __func__, rcr);
 714                }
 715                break;
 716        default:
 717                WARN(1, "Unsupported LUN mode!");
 718                goto out;
 719        }
 720
 721        rhte_checkin(ctxi, rhte);
 722        cxlflash_lun_detach(gli);
 723
 724out:
 725        if (put_ctx)
 726                put_context(ctxi);
 727        dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
 728        return rc;
 729}
 730
 731int cxlflash_disk_release(struct scsi_device *sdev,
 732                          struct dk_cxlflash_release *release)
 733{
 734        return _cxlflash_disk_release(sdev, NULL, release);
 735}
 736
 737/**
 738 * destroy_context() - releases a context
 739 * @cfg:        Internal structure associated with the host.
 740 * @ctxi:       Context to release.
 741 *
 742 * This routine is safe to be called with a a non-initialized context.
 743 * Also note that the routine conditionally checks for the existence
 744 * of the context control map before clearing the RHT registers and
 745 * context capabilities because it is possible to destroy a context
 746 * while the context is in the error state (previous mapping was
 747 * removed [so there is no need to worry about clearing] and context
 748 * is waiting for a new mapping).
 749 */
 750static void destroy_context(struct cxlflash_cfg *cfg,
 751                            struct ctx_info *ctxi)
 752{
 753        struct afu *afu = cfg->afu;
 754
 755        if (ctxi->initialized) {
 756                WARN_ON(!list_empty(&ctxi->luns));
 757
 758                /* Clear RHT registers and drop all capabilities for context */
 759                if (afu->afu_map && ctxi->ctrl_map) {
 760                        writeq_be(0, &ctxi->ctrl_map->rht_start);
 761                        writeq_be(0, &ctxi->ctrl_map->rht_cnt_id);
 762                        writeq_be(0, &ctxi->ctrl_map->ctx_cap);
 763                }
 764        }
 765
 766        /* Free memory associated with context */
 767        free_page((ulong)ctxi->rht_start);
 768        kfree(ctxi->rht_needs_ws);
 769        kfree(ctxi->rht_lun);
 770        kfree(ctxi);
 771}
 772
 773/**
 774 * create_context() - allocates and initializes a context
 775 * @cfg:        Internal structure associated with the host.
 776 *
 777 * Return: Allocated context on success, NULL on failure
 778 */
 779static struct ctx_info *create_context(struct cxlflash_cfg *cfg)
 780{
 781        struct device *dev = &cfg->dev->dev;
 782        struct ctx_info *ctxi = NULL;
 783        struct llun_info **lli = NULL;
 784        u8 *ws = NULL;
 785        struct sisl_rht_entry *rhte;
 786
 787        ctxi = kzalloc(sizeof(*ctxi), GFP_KERNEL);
 788        lli = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*lli)), GFP_KERNEL);
 789        ws = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*ws)), GFP_KERNEL);
 790        if (unlikely(!ctxi || !lli || !ws)) {
 791                dev_err(dev, "%s: Unable to allocate context\n", __func__);
 792                goto err;
 793        }
 794
 795        rhte = (struct sisl_rht_entry *)get_zeroed_page(GFP_KERNEL);
 796        if (unlikely(!rhte)) {
 797                dev_err(dev, "%s: Unable to allocate RHT\n", __func__);
 798                goto err;
 799        }
 800
 801        ctxi->rht_lun = lli;
 802        ctxi->rht_needs_ws = ws;
 803        ctxi->rht_start = rhte;
 804out:
 805        return ctxi;
 806
 807err:
 808        kfree(ws);
 809        kfree(lli);
 810        kfree(ctxi);
 811        ctxi = NULL;
 812        goto out;
 813}
 814
 815/**
 816 * init_context() - initializes a previously allocated context
 817 * @ctxi:       Previously allocated context
 818 * @cfg:        Internal structure associated with the host.
 819 * @ctx:        Previously obtained context cookie.
 820 * @ctxid:      Previously obtained process element associated with CXL context.
 821 * @file:       Previously obtained file associated with CXL context.
 822 * @perms:      User-specified permissions.
 823 * @irqs:       User-specified number of interrupts.
 824 */
 825static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg,
 826                         void *ctx, int ctxid, struct file *file, u32 perms,
 827                         u64 irqs)
 828{
 829        struct afu *afu = cfg->afu;
 830
 831        ctxi->rht_perms = perms;
 832        ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
 833        ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
 834        ctxi->irqs = irqs;
 835        ctxi->pid = task_tgid_nr(current); /* tgid = pid */
 836        ctxi->ctx = ctx;
 837        ctxi->cfg = cfg;
 838        ctxi->file = file;
 839        ctxi->initialized = true;
 840        mutex_init(&ctxi->mutex);
 841        kref_init(&ctxi->kref);
 842        INIT_LIST_HEAD(&ctxi->luns);
 843        INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */
 844}
 845
 846/**
 847 * remove_context() - context kref release handler
 848 * @kref:       Kernel reference associated with context to be removed.
 849 *
 850 * When a context no longer has any references it can safely be removed
 851 * from global access and destroyed. Note that it is assumed the thread
 852 * relinquishing access to the context holds its mutex.
 853 */
 854static void remove_context(struct kref *kref)
 855{
 856        struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref);
 857        struct cxlflash_cfg *cfg = ctxi->cfg;
 858        u64 ctxid = DECODE_CTXID(ctxi->ctxid);
 859
 860        /* Remove context from table/error list */
 861        WARN_ON(!mutex_is_locked(&ctxi->mutex));
 862        ctxi->unavail = true;
 863        mutex_unlock(&ctxi->mutex);
 864        mutex_lock(&cfg->ctx_tbl_list_mutex);
 865        mutex_lock(&ctxi->mutex);
 866
 867        if (!list_empty(&ctxi->list))
 868                list_del(&ctxi->list);
 869        cfg->ctx_tbl[ctxid] = NULL;
 870        mutex_unlock(&cfg->ctx_tbl_list_mutex);
 871        mutex_unlock(&ctxi->mutex);
 872
 873        /* Context now completely uncoupled/unreachable */
 874        destroy_context(cfg, ctxi);
 875}
 876
 877/**
 878 * _cxlflash_disk_detach() - detaches a LUN from a context
 879 * @sdev:       SCSI device associated with LUN.
 880 * @ctxi:       Context owning resources.
 881 * @detach:     Detach ioctl data structure.
 882 *
 883 * As part of the detach, all per-context resources associated with the LUN
 884 * are cleaned up. When detaching the last LUN for a context, the context
 885 * itself is cleaned up and released.
 886 *
 887 * Return: 0 on success, -errno on failure
 888 */
 889static int _cxlflash_disk_detach(struct scsi_device *sdev,
 890                                 struct ctx_info *ctxi,
 891                                 struct dk_cxlflash_detach *detach)
 892{
 893        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
 894        struct device *dev = &cfg->dev->dev;
 895        struct llun_info *lli = sdev->hostdata;
 896        struct lun_access *lun_access, *t;
 897        struct dk_cxlflash_release rel;
 898        bool put_ctx = false;
 899
 900        int i;
 901        int rc = 0;
 902        u64 ctxid = DECODE_CTXID(detach->context_id),
 903            rctxid = detach->context_id;
 904
 905        dev_dbg(dev, "%s: ctxid=%llu\n", __func__, ctxid);
 906
 907        if (!ctxi) {
 908                ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
 909                if (unlikely(!ctxi)) {
 910                        dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
 911                                __func__, ctxid);
 912                        rc = -EINVAL;
 913                        goto out;
 914                }
 915
 916                put_ctx = true;
 917        }
 918
 919        /* Cleanup outstanding resources tied to this LUN */
 920        if (ctxi->rht_out) {
 921                marshal_det_to_rele(detach, &rel);
 922                for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {
 923                        if (ctxi->rht_lun[i] == lli) {
 924                                rel.rsrc_handle = i;
 925                                _cxlflash_disk_release(sdev, ctxi, &rel);
 926                        }
 927
 928                        /* No need to loop further if we're done */
 929                        if (ctxi->rht_out == 0)
 930                                break;
 931                }
 932        }
 933
 934        /* Take our LUN out of context, free the node */
 935        list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
 936                if (lun_access->lli == lli) {
 937                        list_del(&lun_access->list);
 938                        kfree(lun_access);
 939                        lun_access = NULL;
 940                        break;
 941                }
 942
 943        /*
 944         * Release the context reference and the sdev reference that
 945         * bound this LUN to the context.
 946         */
 947        if (kref_put(&ctxi->kref, remove_context))
 948                put_ctx = false;
 949        scsi_device_put(sdev);
 950out:
 951        if (put_ctx)
 952                put_context(ctxi);
 953        dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
 954        return rc;
 955}
 956
 957static int cxlflash_disk_detach(struct scsi_device *sdev,
 958                                struct dk_cxlflash_detach *detach)
 959{
 960        return _cxlflash_disk_detach(sdev, NULL, detach);
 961}
 962
 963/**
 964 * cxlflash_cxl_release() - release handler for adapter file descriptor
 965 * @inode:      File-system inode associated with fd.
 966 * @file:       File installed with adapter file descriptor.
 967 *
 968 * This routine is the release handler for the fops registered with
 969 * the CXL services on an initial attach for a context. It is called
 970 * when a close (explicity by the user or as part of a process tear
 971 * down) is performed on the adapter file descriptor returned to the
 972 * user. The user should be aware that explicitly performing a close
 973 * considered catastrophic and subsequent usage of the superpipe API
 974 * with previously saved off tokens will fail.
 975 *
 976 * This routine derives the context reference and calls detach for
 977 * each LUN associated with the context.The final detach operation
 978 * causes the context itself to be freed. With exception to when the
 979 * CXL process element (context id) lookup fails (a case that should
 980 * theoretically never occur), every call into this routine results
 981 * in a complete freeing of a context.
 982 *
 983 * Detaching the LUN is typically an ioctl() operation and the underlying
 984 * code assumes that ioctl_rwsem has been acquired as a reader. To support
 985 * that design point, the semaphore is acquired and released around detach.
 986 *
 987 * Return: 0 on success
 988 */
 989static int cxlflash_cxl_release(struct inode *inode, struct file *file)
 990{
 991        struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
 992                                                cxl_fops);
 993        void *ctx = cfg->ops->fops_get_context(file);
 994        struct device *dev = &cfg->dev->dev;
 995        struct ctx_info *ctxi = NULL;
 996        struct dk_cxlflash_detach detach = { { 0 }, 0 };
 997        struct lun_access *lun_access, *t;
 998        enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
 999        int ctxid;
1000
1001        ctxid = cfg->ops->process_element(ctx);
1002        if (unlikely(ctxid < 0)) {
1003                dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1004                        __func__, ctx, ctxid);
1005                goto out;
1006        }
1007
1008        ctxi = get_context(cfg, ctxid, file, ctrl);
1009        if (unlikely(!ctxi)) {
1010                ctxi = get_context(cfg, ctxid, file, ctrl | CTX_CTRL_CLONE);
1011                if (!ctxi) {
1012                        dev_dbg(dev, "%s: ctxid=%d already free\n",
1013                                __func__, ctxid);
1014                        goto out_release;
1015                }
1016
1017                dev_dbg(dev, "%s: Another process owns ctxid=%d\n",
1018                        __func__, ctxid);
1019                put_context(ctxi);
1020                goto out;
1021        }
1022
1023        dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);
1024
1025        down_read(&cfg->ioctl_rwsem);
1026        detach.context_id = ctxi->ctxid;
1027        list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
1028                _cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
1029        up_read(&cfg->ioctl_rwsem);
1030out_release:
1031        cfg->ops->fd_release(inode, file);
1032out:
1033        dev_dbg(dev, "%s: returning\n", __func__);
1034        return 0;
1035}
1036
1037/**
1038 * unmap_context() - clears a previously established mapping
1039 * @ctxi:       Context owning the mapping.
1040 *
1041 * This routine is used to switch between the error notification page
1042 * (dummy page of all 1's) and the real mapping (established by the CXL
1043 * fault handler).
1044 */
1045static void unmap_context(struct ctx_info *ctxi)
1046{
1047        unmap_mapping_range(ctxi->file->f_mapping, 0, 0, 1);
1048}
1049
1050/**
1051 * get_err_page() - obtains and allocates the error notification page
1052 * @cfg:        Internal structure associated with the host.
1053 *
1054 * Return: error notification page on success, NULL on failure
1055 */
1056static struct page *get_err_page(struct cxlflash_cfg *cfg)
1057{
1058        struct page *err_page = global.err_page;
1059        struct device *dev = &cfg->dev->dev;
1060
1061        if (unlikely(!err_page)) {
1062                err_page = alloc_page(GFP_KERNEL);
1063                if (unlikely(!err_page)) {
1064                        dev_err(dev, "%s: Unable to allocate err_page\n",
1065                                __func__);
1066                        goto out;
1067                }
1068
1069                memset(page_address(err_page), -1, PAGE_SIZE);
1070
1071                /* Serialize update w/ other threads to avoid a leak */
1072                mutex_lock(&global.mutex);
1073                if (likely(!global.err_page))
1074                        global.err_page = err_page;
1075                else {
1076                        __free_page(err_page);
1077                        err_page = global.err_page;
1078                }
1079                mutex_unlock(&global.mutex);
1080        }
1081
1082out:
1083        dev_dbg(dev, "%s: returning err_page=%p\n", __func__, err_page);
1084        return err_page;
1085}
1086
1087/**
1088 * cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor
1089 * @vmf:        VM fault associated with current fault.
1090 *
1091 * To support error notification via MMIO, faults are 'caught' by this routine
1092 * that was inserted before passing back the adapter file descriptor on attach.
1093 * When a fault occurs, this routine evaluates if error recovery is active and
1094 * if so, installs the error page to 'notify' the user about the error state.
1095 * During normal operation, the fault is simply handled by the original fault
1096 * handler that was installed by CXL services as part of initializing the
1097 * adapter file descriptor. The VMA's page protection bits are toggled to
1098 * indicate cached/not-cached depending on the memory backing the fault.
1099 *
1100 * Return: 0 on success, VM_FAULT_SIGBUS on failure
1101 */
1102static vm_fault_t cxlflash_mmap_fault(struct vm_fault *vmf)
1103{
1104        struct vm_area_struct *vma = vmf->vma;
1105        struct file *file = vma->vm_file;
1106        struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
1107                                                cxl_fops);
1108        void *ctx = cfg->ops->fops_get_context(file);
1109        struct device *dev = &cfg->dev->dev;
1110        struct ctx_info *ctxi = NULL;
1111        struct page *err_page = NULL;
1112        enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
1113        vm_fault_t rc = 0;
1114        int ctxid;
1115
1116        ctxid = cfg->ops->process_element(ctx);
1117        if (unlikely(ctxid < 0)) {
1118                dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1119                        __func__, ctx, ctxid);
1120                goto err;
1121        }
1122
1123        ctxi = get_context(cfg, ctxid, file, ctrl);
1124        if (unlikely(!ctxi)) {
1125                dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
1126                goto err;
1127        }
1128
1129        dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid);
1130
1131        if (likely(!ctxi->err_recovery_active)) {
1132                vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1133                rc = ctxi->cxl_mmap_vmops->fault(vmf);
1134        } else {
1135                dev_dbg(dev, "%s: err recovery active, use err_page\n",
1136                        __func__);
1137
1138                err_page = get_err_page(cfg);
1139                if (unlikely(!err_page)) {
1140                        dev_err(dev, "%s: Could not get err_page\n", __func__);
1141                        rc = VM_FAULT_RETRY;
1142                        goto out;
1143                }
1144
1145                get_page(err_page);
1146                vmf->page = err_page;
1147                vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
1148        }
1149
1150out:
1151        if (likely(ctxi))
1152                put_context(ctxi);
1153        dev_dbg(dev, "%s: returning rc=%x\n", __func__, rc);
1154        return rc;
1155
1156err:
1157        rc = VM_FAULT_SIGBUS;
1158        goto out;
1159}
1160
1161/*
1162 * Local MMAP vmops to 'catch' faults
1163 */
1164static const struct vm_operations_struct cxlflash_mmap_vmops = {
1165        .fault = cxlflash_mmap_fault,
1166};
1167
1168/**
1169 * cxlflash_cxl_mmap() - mmap handler for adapter file descriptor
1170 * @file:       File installed with adapter file descriptor.
1171 * @vma:        VM area associated with mapping.
1172 *
1173 * Installs local mmap vmops to 'catch' faults for error notification support.
1174 *
1175 * Return: 0 on success, -errno on failure
1176 */
1177static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma)
1178{
1179        struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
1180                                                cxl_fops);
1181        void *ctx = cfg->ops->fops_get_context(file);
1182        struct device *dev = &cfg->dev->dev;
1183        struct ctx_info *ctxi = NULL;
1184        enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
1185        int ctxid;
1186        int rc = 0;
1187
1188        ctxid = cfg->ops->process_element(ctx);
1189        if (unlikely(ctxid < 0)) {
1190                dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1191                        __func__, ctx, ctxid);
1192                rc = -EIO;
1193                goto out;
1194        }
1195
1196        ctxi = get_context(cfg, ctxid, file, ctrl);
1197        if (unlikely(!ctxi)) {
1198                dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
1199                rc = -EIO;
1200                goto out;
1201        }
1202
1203        dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid);
1204
1205        rc = cfg->ops->fd_mmap(file, vma);
1206        if (likely(!rc)) {
1207                /* Insert ourself in the mmap fault handler path */
1208                ctxi->cxl_mmap_vmops = vma->vm_ops;
1209                vma->vm_ops = &cxlflash_mmap_vmops;
1210        }
1211
1212out:
1213        if (likely(ctxi))
1214                put_context(ctxi);
1215        return rc;
1216}
1217
1218const struct file_operations cxlflash_cxl_fops = {
1219        .owner = THIS_MODULE,
1220        .mmap = cxlflash_cxl_mmap,
1221        .release = cxlflash_cxl_release,
1222};
1223
1224/**
1225 * cxlflash_mark_contexts_error() - move contexts to error state and list
1226 * @cfg:        Internal structure associated with the host.
1227 *
1228 * A context is only moved over to the error list when there are no outstanding
1229 * references to it. This ensures that a running operation has completed.
1230 *
1231 * Return: 0 on success, -errno on failure
1232 */
1233int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg)
1234{
1235        int i, rc = 0;
1236        struct ctx_info *ctxi = NULL;
1237
1238        mutex_lock(&cfg->ctx_tbl_list_mutex);
1239
1240        for (i = 0; i < MAX_CONTEXT; i++) {
1241                ctxi = cfg->ctx_tbl[i];
1242                if (ctxi) {
1243                        mutex_lock(&ctxi->mutex);
1244                        cfg->ctx_tbl[i] = NULL;
1245                        list_add(&ctxi->list, &cfg->ctx_err_recovery);
1246                        ctxi->err_recovery_active = true;
1247                        ctxi->ctrl_map = NULL;
1248                        unmap_context(ctxi);
1249                        mutex_unlock(&ctxi->mutex);
1250                }
1251        }
1252
1253        mutex_unlock(&cfg->ctx_tbl_list_mutex);
1254        return rc;
1255}
1256
1257/*
1258 * Dummy NULL fops
1259 */
1260static const struct file_operations null_fops = {
1261        .owner = THIS_MODULE,
1262};
1263
1264/**
1265 * check_state() - checks and responds to the current adapter state
1266 * @cfg:        Internal structure associated with the host.
1267 *
1268 * This routine can block and should only be used on process context.
1269 * It assumes that the caller is an ioctl thread and holding the ioctl
1270 * read semaphore. This is temporarily let up across the wait to allow
1271 * for draining actively running ioctls. Also note that when waking up
1272 * from waiting in reset, the state is unknown and must be checked again
1273 * before proceeding.
1274 *
1275 * Return: 0 on success, -errno on failure
1276 */
1277int check_state(struct cxlflash_cfg *cfg)
1278{
1279        struct device *dev = &cfg->dev->dev;
1280        int rc = 0;
1281
1282retry:
1283        switch (cfg->state) {
1284        case STATE_RESET:
1285                dev_dbg(dev, "%s: Reset state, going to wait...\n", __func__);
1286                up_read(&cfg->ioctl_rwsem);
1287                rc = wait_event_interruptible(cfg->reset_waitq,
1288                                              cfg->state != STATE_RESET);
1289                down_read(&cfg->ioctl_rwsem);
1290                if (unlikely(rc))
1291                        break;
1292                goto retry;
1293        case STATE_FAILTERM:
1294                dev_dbg(dev, "%s: Failed/Terminating\n", __func__);
1295                rc = -ENODEV;
1296                break;
1297        default:
1298                break;
1299        }
1300
1301        return rc;
1302}
1303
1304/**
1305 * cxlflash_disk_attach() - attach a LUN to a context
1306 * @sdev:       SCSI device associated with LUN.
1307 * @attach:     Attach ioctl data structure.
1308 *
1309 * Creates a context and attaches LUN to it. A LUN can only be attached
1310 * one time to a context (subsequent attaches for the same context/LUN pair
1311 * are not supported). Additional LUNs can be attached to a context by
1312 * specifying the 'reuse' flag defined in the cxlflash_ioctl.h header.
1313 *
1314 * Return: 0 on success, -errno on failure
1315 */
1316static int cxlflash_disk_attach(struct scsi_device *sdev,
1317                                struct dk_cxlflash_attach *attach)
1318{
1319        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1320        struct device *dev = &cfg->dev->dev;
1321        struct afu *afu = cfg->afu;
1322        struct llun_info *lli = sdev->hostdata;
1323        struct glun_info *gli = lli->parent;
1324        struct ctx_info *ctxi = NULL;
1325        struct lun_access *lun_access = NULL;
1326        int rc = 0;
1327        u32 perms;
1328        int ctxid = -1;
1329        u64 irqs = attach->num_interrupts;
1330        u64 flags = 0UL;
1331        u64 rctxid = 0UL;
1332        struct file *file = NULL;
1333
1334        void *ctx = NULL;
1335
1336        int fd = -1;
1337
1338        if (irqs > 4) {
1339                dev_dbg(dev, "%s: Cannot support this many interrupts %llu\n",
1340                        __func__, irqs);
1341                rc = -EINVAL;
1342                goto out;
1343        }
1344
1345        if (gli->max_lba == 0) {
1346                dev_dbg(dev, "%s: No capacity info for LUN=%016llx\n",
1347                        __func__, lli->lun_id[sdev->channel]);
1348                rc = read_cap16(sdev, lli);
1349                if (rc) {
1350                        dev_err(dev, "%s: Invalid device rc=%d\n",
1351                                __func__, rc);
1352                        rc = -ENODEV;
1353                        goto out;
1354                }
1355                dev_dbg(dev, "%s: LBA = %016llx\n", __func__, gli->max_lba);
1356                dev_dbg(dev, "%s: BLK_LEN = %08x\n", __func__, gli->blk_len);
1357        }
1358
1359        if (attach->hdr.flags & DK_CXLFLASH_ATTACH_REUSE_CONTEXT) {
1360                rctxid = attach->context_id;
1361                ctxi = get_context(cfg, rctxid, NULL, 0);
1362                if (!ctxi) {
1363                        dev_dbg(dev, "%s: Bad context rctxid=%016llx\n",
1364                                __func__, rctxid);
1365                        rc = -EINVAL;
1366                        goto out;
1367                }
1368
1369                list_for_each_entry(lun_access, &ctxi->luns, list)
1370                        if (lun_access->lli == lli) {
1371                                dev_dbg(dev, "%s: Already attached\n",
1372                                        __func__);
1373                                rc = -EINVAL;
1374                                goto out;
1375                        }
1376        }
1377
1378        rc = scsi_device_get(sdev);
1379        if (unlikely(rc)) {
1380                dev_err(dev, "%s: Unable to get sdev reference\n", __func__);
1381                goto out;
1382        }
1383
1384        lun_access = kzalloc(sizeof(*lun_access), GFP_KERNEL);
1385        if (unlikely(!lun_access)) {
1386                dev_err(dev, "%s: Unable to allocate lun_access\n", __func__);
1387                rc = -ENOMEM;
1388                goto err;
1389        }
1390
1391        lun_access->lli = lli;
1392        lun_access->sdev = sdev;
1393
1394        /* Non-NULL context indicates reuse (another context reference) */
1395        if (ctxi) {
1396                dev_dbg(dev, "%s: Reusing context for LUN rctxid=%016llx\n",
1397                        __func__, rctxid);
1398                kref_get(&ctxi->kref);
1399                list_add(&lun_access->list, &ctxi->luns);
1400                goto out_attach;
1401        }
1402
1403        ctxi = create_context(cfg);
1404        if (unlikely(!ctxi)) {
1405                dev_err(dev, "%s: Failed to create context ctxid=%d\n",
1406                        __func__, ctxid);
1407                rc = -ENOMEM;
1408                goto err;
1409        }
1410
1411        ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
1412        if (IS_ERR_OR_NULL(ctx)) {
1413                dev_err(dev, "%s: Could not initialize context %p\n",
1414                        __func__, ctx);
1415                rc = -ENODEV;
1416                goto err;
1417        }
1418
1419        rc = cfg->ops->start_work(ctx, irqs);
1420        if (unlikely(rc)) {
1421                dev_dbg(dev, "%s: Could not start context rc=%d\n",
1422                        __func__, rc);
1423                goto err;
1424        }
1425
1426        ctxid = cfg->ops->process_element(ctx);
1427        if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
1428                dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
1429                rc = -EPERM;
1430                goto err;
1431        }
1432
1433        file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);
1434        if (unlikely(fd < 0)) {
1435                rc = -ENODEV;
1436                dev_err(dev, "%s: Could not get file descriptor\n", __func__);
1437                goto err;
1438        }
1439
1440        /* Translate read/write O_* flags from fcntl.h to AFU permission bits */
1441        perms = SISL_RHT_PERM(attach->hdr.flags + 1);
1442
1443        /* Context mutex is locked upon return */
1444        init_context(ctxi, cfg, ctx, ctxid, file, perms, irqs);
1445
1446        rc = afu_attach(cfg, ctxi);
1447        if (unlikely(rc)) {
1448                dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
1449                goto err;
1450        }
1451
1452        /*
1453         * No error paths after this point. Once the fd is installed it's
1454         * visible to user space and can't be undone safely on this thread.
1455         * There is no need to worry about a deadlock here because no one
1456         * knows about us yet; we can be the only one holding our mutex.
1457         */
1458        list_add(&lun_access->list, &ctxi->luns);
1459        mutex_lock(&cfg->ctx_tbl_list_mutex);
1460        mutex_lock(&ctxi->mutex);
1461        cfg->ctx_tbl[ctxid] = ctxi;
1462        mutex_unlock(&cfg->ctx_tbl_list_mutex);
1463        fd_install(fd, file);
1464
1465out_attach:
1466        if (fd != -1)
1467                flags |= DK_CXLFLASH_APP_CLOSE_ADAP_FD;
1468        if (afu_is_sq_cmd_mode(afu))
1469                flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
1470
1471        attach->hdr.return_flags = flags;
1472        attach->context_id = ctxi->ctxid;
1473        attach->block_size = gli->blk_len;
1474        attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
1475        attach->last_lba = gli->max_lba;
1476        attach->max_xfer = sdev->host->max_sectors * MAX_SECTOR_UNIT;
1477        attach->max_xfer /= gli->blk_len;
1478
1479out:
1480        attach->adap_fd = fd;
1481
1482        if (ctxi)
1483                put_context(ctxi);
1484
1485        dev_dbg(dev, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n",
1486                __func__, ctxid, fd, attach->block_size, rc, attach->last_lba);
1487        return rc;
1488
1489err:
1490        /* Cleanup CXL context; okay to 'stop' even if it was not started */
1491        if (!IS_ERR_OR_NULL(ctx)) {
1492                cfg->ops->stop_context(ctx);
1493                cfg->ops->release_context(ctx);
1494                ctx = NULL;
1495        }
1496
1497        /*
1498         * Here, we're overriding the fops with a dummy all-NULL fops because
1499         * fput() calls the release fop, which will cause us to mistakenly
1500         * call into the CXL code. Rather than try to add yet more complexity
1501         * to that routine (cxlflash_cxl_release) we should try to fix the
1502         * issue here.
1503         */
1504        if (fd > 0) {
1505                file->f_op = &null_fops;
1506                fput(file);
1507                put_unused_fd(fd);
1508                fd = -1;
1509                file = NULL;
1510        }
1511
1512        /* Cleanup our context */
1513        if (ctxi) {
1514                destroy_context(cfg, ctxi);
1515                ctxi = NULL;
1516        }
1517
1518        kfree(lun_access);
1519        scsi_device_put(sdev);
1520        goto out;
1521}
1522
1523/**
1524 * recover_context() - recovers a context in error
1525 * @cfg:        Internal structure associated with the host.
1526 * @ctxi:       Context to release.
1527 * @adap_fd:    Adapter file descriptor associated with new/recovered context.
1528 *
1529 * Restablishes the state for a context-in-error.
1530 *
1531 * Return: 0 on success, -errno on failure
1532 */
1533static int recover_context(struct cxlflash_cfg *cfg,
1534                           struct ctx_info *ctxi,
1535                           int *adap_fd)
1536{
1537        struct device *dev = &cfg->dev->dev;
1538        int rc = 0;
1539        int fd = -1;
1540        int ctxid = -1;
1541        struct file *file;
1542        void *ctx;
1543        struct afu *afu = cfg->afu;
1544
1545        ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
1546        if (IS_ERR_OR_NULL(ctx)) {
1547                dev_err(dev, "%s: Could not initialize context %p\n",
1548                        __func__, ctx);
1549                rc = -ENODEV;
1550                goto out;
1551        }
1552
1553        rc = cfg->ops->start_work(ctx, ctxi->irqs);
1554        if (unlikely(rc)) {
1555                dev_dbg(dev, "%s: Could not start context rc=%d\n",
1556                        __func__, rc);
1557                goto err1;
1558        }
1559
1560        ctxid = cfg->ops->process_element(ctx);
1561        if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
1562                dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
1563                rc = -EPERM;
1564                goto err2;
1565        }
1566
1567        file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);
1568        if (unlikely(fd < 0)) {
1569                rc = -ENODEV;
1570                dev_err(dev, "%s: Could not get file descriptor\n", __func__);
1571                goto err2;
1572        }
1573
1574        /* Update with new MMIO area based on updated context id */
1575        ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
1576
1577        rc = afu_attach(cfg, ctxi);
1578        if (rc) {
1579                dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
1580                goto err3;
1581        }
1582
1583        /*
1584         * No error paths after this point. Once the fd is installed it's
1585         * visible to user space and can't be undone safely on this thread.
1586         */
1587        ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
1588        ctxi->ctx = ctx;
1589        ctxi->file = file;
1590
1591        /*
1592         * Put context back in table (note the reinit of the context list);
1593         * we must first drop the context's mutex and then acquire it in
1594         * order with the table/list mutex to avoid a deadlock - safe to do
1595         * here because no one can find us at this moment in time.
1596         */
1597        mutex_unlock(&ctxi->mutex);
1598        mutex_lock(&cfg->ctx_tbl_list_mutex);
1599        mutex_lock(&ctxi->mutex);
1600        list_del_init(&ctxi->list);
1601        cfg->ctx_tbl[ctxid] = ctxi;
1602        mutex_unlock(&cfg->ctx_tbl_list_mutex);
1603        fd_install(fd, file);
1604        *adap_fd = fd;
1605out:
1606        dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n",
1607                __func__, ctxid, fd, rc);
1608        return rc;
1609
1610err3:
1611        fput(file);
1612        put_unused_fd(fd);
1613err2:
1614        cfg->ops->stop_context(ctx);
1615err1:
1616        cfg->ops->release_context(ctx);
1617        goto out;
1618}
1619
1620/**
1621 * cxlflash_afu_recover() - initiates AFU recovery
1622 * @sdev:       SCSI device associated with LUN.
1623 * @recover:    Recover ioctl data structure.
1624 *
1625 * Only a single recovery is allowed at a time to avoid exhausting CXL
1626 * resources (leading to recovery failure) in the event that we're up
1627 * against the maximum number of contexts limit. For similar reasons,
1628 * a context recovery is retried if there are multiple recoveries taking
1629 * place at the same time and the failure was due to CXL services being
1630 * unable to keep up.
1631 *
1632 * As this routine is called on ioctl context, it holds the ioctl r/w
1633 * semaphore that is used to drain ioctls in recovery scenarios. The
1634 * implementation to achieve the pacing described above (a local mutex)
1635 * requires that the ioctl r/w semaphore be dropped and reacquired to
1636 * avoid a 3-way deadlock when multiple process recoveries operate in
1637 * parallel.
1638 *
1639 * Because a user can detect an error condition before the kernel, it is
1640 * quite possible for this routine to act as the kernel's EEH detection
1641 * source (MMIO read of mbox_r). Because of this, there is a window of
1642 * time where an EEH might have been detected but not yet 'serviced'
1643 * (callback invoked, causing the device to enter reset state). To avoid
1644 * looping in this routine during that window, a 1 second sleep is in place
1645 * between the time the MMIO failure is detected and the time a wait on the
1646 * reset wait queue is attempted via check_state().
1647 *
1648 * Return: 0 on success, -errno on failure
1649 */
1650static int cxlflash_afu_recover(struct scsi_device *sdev,
1651                                struct dk_cxlflash_recover_afu *recover)
1652{
1653        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1654        struct device *dev = &cfg->dev->dev;
1655        struct llun_info *lli = sdev->hostdata;
1656        struct afu *afu = cfg->afu;
1657        struct ctx_info *ctxi = NULL;
1658        struct mutex *mutex = &cfg->ctx_recovery_mutex;
1659        struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
1660        u64 flags;
1661        u64 ctxid = DECODE_CTXID(recover->context_id),
1662            rctxid = recover->context_id;
1663        long reg;
1664        bool locked = true;
1665        int lretry = 20; /* up to 2 seconds */
1666        int new_adap_fd = -1;
1667        int rc = 0;
1668
1669        atomic_inc(&cfg->recovery_threads);
1670        up_read(&cfg->ioctl_rwsem);
1671        rc = mutex_lock_interruptible(mutex);
1672        down_read(&cfg->ioctl_rwsem);
1673        if (rc) {
1674                locked = false;
1675                goto out;
1676        }
1677
1678        rc = check_state(cfg);
1679        if (rc) {
1680                dev_err(dev, "%s: Failed state rc=%d\n", __func__, rc);
1681                rc = -ENODEV;
1682                goto out;
1683        }
1684
1685        dev_dbg(dev, "%s: reason=%016llx rctxid=%016llx\n",
1686                __func__, recover->reason, rctxid);
1687
1688retry:
1689        /* Ensure that this process is attached to the context */
1690        ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
1691        if (unlikely(!ctxi)) {
1692                dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1693                rc = -EINVAL;
1694                goto out;
1695        }
1696
1697        if (ctxi->err_recovery_active) {
1698retry_recover:
1699                rc = recover_context(cfg, ctxi, &new_adap_fd);
1700                if (unlikely(rc)) {
1701                        dev_err(dev, "%s: Recovery failed ctxid=%llu rc=%d\n",
1702                                __func__, ctxid, rc);
1703                        if ((rc == -ENODEV) &&
1704                            ((atomic_read(&cfg->recovery_threads) > 1) ||
1705                             (lretry--))) {
1706                                dev_dbg(dev, "%s: Going to try again\n",
1707                                        __func__);
1708                                mutex_unlock(mutex);
1709                                msleep(100);
1710                                rc = mutex_lock_interruptible(mutex);
1711                                if (rc) {
1712                                        locked = false;
1713                                        goto out;
1714                                }
1715                                goto retry_recover;
1716                        }
1717
1718                        goto out;
1719                }
1720
1721                ctxi->err_recovery_active = false;
1722
1723                flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD |
1724                        DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET;
1725                if (afu_is_sq_cmd_mode(afu))
1726                        flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
1727
1728                recover->hdr.return_flags = flags;
1729                recover->context_id = ctxi->ctxid;
1730                recover->adap_fd = new_adap_fd;
1731                recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
1732                goto out;
1733        }
1734
1735        /* Test if in error state */
1736        reg = readq_be(&hwq->ctrl_map->mbox_r);
1737        if (reg == -1) {
1738                dev_dbg(dev, "%s: MMIO fail, wait for recovery.\n", __func__);
1739
1740                /*
1741                 * Before checking the state, put back the context obtained with
1742                 * get_context() as it is no longer needed and sleep for a short
1743                 * period of time (see prolog notes).
1744                 */
1745                put_context(ctxi);
1746                ctxi = NULL;
1747                ssleep(1);
1748                rc = check_state(cfg);
1749                if (unlikely(rc))
1750                        goto out;
1751                goto retry;
1752        }
1753
1754        dev_dbg(dev, "%s: MMIO working, no recovery required\n", __func__);
1755out:
1756        if (likely(ctxi))
1757                put_context(ctxi);
1758        if (locked)
1759                mutex_unlock(mutex);
1760        atomic_dec_if_positive(&cfg->recovery_threads);
1761        return rc;
1762}
1763
1764/**
1765 * process_sense() - evaluates and processes sense data
1766 * @sdev:       SCSI device associated with LUN.
1767 * @verify:     Verify ioctl data structure.
1768 *
1769 * Return: 0 on success, -errno on failure
1770 */
1771static int process_sense(struct scsi_device *sdev,
1772                         struct dk_cxlflash_verify *verify)
1773{
1774        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1775        struct device *dev = &cfg->dev->dev;
1776        struct llun_info *lli = sdev->hostdata;
1777        struct glun_info *gli = lli->parent;
1778        u64 prev_lba = gli->max_lba;
1779        struct scsi_sense_hdr sshdr = { 0 };
1780        int rc = 0;
1781
1782        rc = scsi_normalize_sense((const u8 *)&verify->sense_data,
1783                                  DK_CXLFLASH_VERIFY_SENSE_LEN, &sshdr);
1784        if (!rc) {
1785                dev_err(dev, "%s: Failed to normalize sense data\n", __func__);
1786                rc = -EINVAL;
1787                goto out;
1788        }
1789
1790        switch (sshdr.sense_key) {
1791        case NO_SENSE:
1792        case RECOVERED_ERROR:
1793        case NOT_READY:
1794                break;
1795        case UNIT_ATTENTION:
1796                switch (sshdr.asc) {
1797                case 0x29: /* Power on Reset or Device Reset */
1798                        fallthrough;
1799                case 0x2A: /* Device settings/capacity changed */
1800                        rc = read_cap16(sdev, lli);
1801                        if (rc) {
1802                                rc = -ENODEV;
1803                                break;
1804                        }
1805                        if (prev_lba != gli->max_lba)
1806                                dev_dbg(dev, "%s: Capacity changed old=%lld "
1807                                        "new=%lld\n", __func__, prev_lba,
1808                                        gli->max_lba);
1809                        break;
1810                case 0x3F: /* Report LUNs changed, Rescan. */
1811                        scsi_scan_host(cfg->host);
1812                        break;
1813                default:
1814                        rc = -EIO;
1815                        break;
1816                }
1817                break;
1818        default:
1819                rc = -EIO;
1820                break;
1821        }
1822out:
1823        dev_dbg(dev, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__,
1824                sshdr.sense_key, sshdr.asc, sshdr.ascq, rc);
1825        return rc;
1826}
1827
1828/**
1829 * cxlflash_disk_verify() - verifies a LUN is the same and handle size changes
1830 * @sdev:       SCSI device associated with LUN.
1831 * @verify:     Verify ioctl data structure.
1832 *
1833 * Return: 0 on success, -errno on failure
1834 */
1835static int cxlflash_disk_verify(struct scsi_device *sdev,
1836                                struct dk_cxlflash_verify *verify)
1837{
1838        int rc = 0;
1839        struct ctx_info *ctxi = NULL;
1840        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1841        struct device *dev = &cfg->dev->dev;
1842        struct llun_info *lli = sdev->hostdata;
1843        struct glun_info *gli = lli->parent;
1844        struct sisl_rht_entry *rhte = NULL;
1845        res_hndl_t rhndl = verify->rsrc_handle;
1846        u64 ctxid = DECODE_CTXID(verify->context_id),
1847            rctxid = verify->context_id;
1848        u64 last_lba = 0;
1849
1850        dev_dbg(dev, "%s: ctxid=%llu rhndl=%016llx, hint=%016llx, "
1851                "flags=%016llx\n", __func__, ctxid, verify->rsrc_handle,
1852                verify->hint, verify->hdr.flags);
1853
1854        ctxi = get_context(cfg, rctxid, lli, 0);
1855        if (unlikely(!ctxi)) {
1856                dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1857                rc = -EINVAL;
1858                goto out;
1859        }
1860
1861        rhte = get_rhte(ctxi, rhndl, lli);
1862        if (unlikely(!rhte)) {
1863                dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
1864                        __func__, rhndl);
1865                rc = -EINVAL;
1866                goto out;
1867        }
1868
1869        /*
1870         * Look at the hint/sense to see if it requires us to redrive
1871         * inquiry (i.e. the Unit attention is due to the WWN changing).
1872         */
1873        if (verify->hint & DK_CXLFLASH_VERIFY_HINT_SENSE) {
1874                /* Can't hold mutex across process_sense/read_cap16,
1875                 * since we could have an intervening EEH event.
1876                 */
1877                ctxi->unavail = true;
1878                mutex_unlock(&ctxi->mutex);
1879                rc = process_sense(sdev, verify);
1880                if (unlikely(rc)) {
1881                        dev_err(dev, "%s: Failed to validate sense data (%d)\n",
1882                                __func__, rc);
1883                        mutex_lock(&ctxi->mutex);
1884                        ctxi->unavail = false;
1885                        goto out;
1886                }
1887                mutex_lock(&ctxi->mutex);
1888                ctxi->unavail = false;
1889        }
1890
1891        switch (gli->mode) {
1892        case MODE_PHYSICAL:
1893                last_lba = gli->max_lba;
1894                break;
1895        case MODE_VIRTUAL:
1896                /* Cast lxt_cnt to u64 for multiply to be treated as 64bit op */
1897                last_lba = ((u64)rhte->lxt_cnt * MC_CHUNK_SIZE * gli->blk_len);
1898                last_lba /= CXLFLASH_BLOCK_SIZE;
1899                last_lba--;
1900                break;
1901        default:
1902                WARN(1, "Unsupported LUN mode!");
1903        }
1904
1905        verify->last_lba = last_lba;
1906
1907out:
1908        if (likely(ctxi))
1909                put_context(ctxi);
1910        dev_dbg(dev, "%s: returning rc=%d llba=%llx\n",
1911                __func__, rc, verify->last_lba);
1912        return rc;
1913}
1914
1915/**
1916 * decode_ioctl() - translates an encoded ioctl to an easily identifiable string
1917 * @cmd:        The ioctl command to decode.
1918 *
1919 * Return: A string identifying the decoded ioctl.
1920 */
1921static char *decode_ioctl(unsigned int cmd)
1922{
1923        switch (cmd) {
1924        case DK_CXLFLASH_ATTACH:
1925                return __stringify_1(DK_CXLFLASH_ATTACH);
1926        case DK_CXLFLASH_USER_DIRECT:
1927                return __stringify_1(DK_CXLFLASH_USER_DIRECT);
1928        case DK_CXLFLASH_USER_VIRTUAL:
1929                return __stringify_1(DK_CXLFLASH_USER_VIRTUAL);
1930        case DK_CXLFLASH_VLUN_RESIZE:
1931                return __stringify_1(DK_CXLFLASH_VLUN_RESIZE);
1932        case DK_CXLFLASH_RELEASE:
1933                return __stringify_1(DK_CXLFLASH_RELEASE);
1934        case DK_CXLFLASH_DETACH:
1935                return __stringify_1(DK_CXLFLASH_DETACH);
1936        case DK_CXLFLASH_VERIFY:
1937                return __stringify_1(DK_CXLFLASH_VERIFY);
1938        case DK_CXLFLASH_VLUN_CLONE:
1939                return __stringify_1(DK_CXLFLASH_VLUN_CLONE);
1940        case DK_CXLFLASH_RECOVER_AFU:
1941                return __stringify_1(DK_CXLFLASH_RECOVER_AFU);
1942        case DK_CXLFLASH_MANAGE_LUN:
1943                return __stringify_1(DK_CXLFLASH_MANAGE_LUN);
1944        }
1945
1946        return "UNKNOWN";
1947}
1948
1949/**
1950 * cxlflash_disk_direct_open() - opens a direct (physical) disk
1951 * @sdev:       SCSI device associated with LUN.
1952 * @arg:        UDirect ioctl data structure.
1953 *
1954 * On successful return, the user is informed of the resource handle
1955 * to be used to identify the direct lun and the size (in blocks) of
1956 * the direct lun in last LBA format.
1957 *
1958 * Return: 0 on success, -errno on failure
1959 */
1960static int cxlflash_disk_direct_open(struct scsi_device *sdev, void *arg)
1961{
1962        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1963        struct device *dev = &cfg->dev->dev;
1964        struct afu *afu = cfg->afu;
1965        struct llun_info *lli = sdev->hostdata;
1966        struct glun_info *gli = lli->parent;
1967        struct dk_cxlflash_release rel = { { 0 }, 0 };
1968
1969        struct dk_cxlflash_udirect *pphys = (struct dk_cxlflash_udirect *)arg;
1970
1971        u64 ctxid = DECODE_CTXID(pphys->context_id),
1972            rctxid = pphys->context_id;
1973        u64 lun_size = 0;
1974        u64 last_lba = 0;
1975        u64 rsrc_handle = -1;
1976        u32 port = CHAN2PORTMASK(sdev->channel);
1977
1978        int rc = 0;
1979
1980        struct ctx_info *ctxi = NULL;
1981        struct sisl_rht_entry *rhte = NULL;
1982
1983        dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);
1984
1985        rc = cxlflash_lun_attach(gli, MODE_PHYSICAL, false);
1986        if (unlikely(rc)) {
1987                dev_dbg(dev, "%s: Failed attach to LUN (PHYSICAL)\n", __func__);
1988                goto out;
1989        }
1990
1991        ctxi = get_context(cfg, rctxid, lli, 0);
1992        if (unlikely(!ctxi)) {
1993                dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1994                rc = -EINVAL;
1995                goto err1;
1996        }
1997
1998        rhte = rhte_checkout(ctxi, lli);
1999        if (unlikely(!rhte)) {
2000                dev_dbg(dev, "%s: Too many opens ctxid=%lld\n",
2001                        __func__, ctxid);
2002                rc = -EMFILE;   /* too many opens  */
2003                goto err1;
2004        }
2005
2006        rsrc_handle = (rhte - ctxi->rht_start);
2007
2008        rht_format1(rhte, lli->lun_id[sdev->channel], ctxi->rht_perms, port);
2009
2010        last_lba = gli->max_lba;
2011        pphys->hdr.return_flags = 0;
2012        pphys->last_lba = last_lba;
2013        pphys->rsrc_handle = rsrc_handle;
2014
2015        rc = cxlflash_afu_sync(afu, ctxid, rsrc_handle, AFU_LW_SYNC);
2016        if (unlikely(rc)) {
2017                dev_dbg(dev, "%s: AFU sync failed rc=%d\n", __func__, rc);
2018                goto err2;
2019        }
2020
2021out:
2022        if (likely(ctxi))
2023                put_context(ctxi);
2024        dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",
2025                __func__, rsrc_handle, rc, last_lba);
2026        return rc;
2027
2028err2:
2029        marshal_udir_to_rele(pphys, &rel);
2030        _cxlflash_disk_release(sdev, ctxi, &rel);
2031        goto out;
2032err1:
2033        cxlflash_lun_detach(gli);
2034        goto out;
2035}
2036
2037/**
2038 * ioctl_common() - common IOCTL handler for driver
2039 * @sdev:       SCSI device associated with LUN.
2040 * @cmd:        IOCTL command.
2041 *
2042 * Handles common fencing operations that are valid for multiple ioctls. Always
2043 * allow through ioctls that are cleanup oriented in nature, even when operating
2044 * in a failed/terminating state.
2045 *
2046 * Return: 0 on success, -errno on failure
2047 */
2048static int ioctl_common(struct scsi_device *sdev, unsigned int cmd)
2049{
2050        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
2051        struct device *dev = &cfg->dev->dev;
2052        struct llun_info *lli = sdev->hostdata;
2053        int rc = 0;
2054
2055        if (unlikely(!lli)) {
2056                dev_dbg(dev, "%s: Unknown LUN\n", __func__);
2057                rc = -EINVAL;
2058                goto out;
2059        }
2060
2061        rc = check_state(cfg);
2062        if (unlikely(rc) && (cfg->state == STATE_FAILTERM)) {
2063                switch (cmd) {
2064                case DK_CXLFLASH_VLUN_RESIZE:
2065                case DK_CXLFLASH_RELEASE:
2066                case DK_CXLFLASH_DETACH:
2067                        dev_dbg(dev, "%s: Command override rc=%d\n",
2068                                __func__, rc);
2069                        rc = 0;
2070                        break;
2071                }
2072        }
2073out:
2074        return rc;
2075}
2076
2077/**
2078 * cxlflash_ioctl() - IOCTL handler for driver
2079 * @sdev:       SCSI device associated with LUN.
2080 * @cmd:        IOCTL command.
2081 * @arg:        Userspace ioctl data structure.
2082 *
2083 * A read/write semaphore is used to implement a 'drain' of currently
2084 * running ioctls. The read semaphore is taken at the beginning of each
2085 * ioctl thread and released upon concluding execution. Additionally the
2086 * semaphore should be released and then reacquired in any ioctl execution
2087 * path which will wait for an event to occur that is outside the scope of
2088 * the ioctl (i.e. an adapter reset). To drain the ioctls currently running,
2089 * a thread simply needs to acquire the write semaphore.
2090 *
2091 * Return: 0 on success, -errno on failure
2092 */
2093int cxlflash_ioctl(struct scsi_device *sdev, unsigned int cmd, void __user *arg)
2094{
2095        typedef int (*sioctl) (struct scsi_device *, void *);
2096
2097        struct cxlflash_cfg *cfg = shost_priv(sdev->host);
2098        struct device *dev = &cfg->dev->dev;
2099        struct afu *afu = cfg->afu;
2100        struct dk_cxlflash_hdr *hdr;
2101        char buf[sizeof(union cxlflash_ioctls)];
2102        size_t size = 0;
2103        bool known_ioctl = false;
2104        int idx;
2105        int rc = 0;
2106        struct Scsi_Host *shost = sdev->host;
2107        sioctl do_ioctl = NULL;
2108
2109        static const struct {
2110                size_t size;
2111                sioctl ioctl;
2112        } ioctl_tbl[] = {       /* NOTE: order matters here */
2113        {sizeof(struct dk_cxlflash_attach), (sioctl)cxlflash_disk_attach},
2114        {sizeof(struct dk_cxlflash_udirect), cxlflash_disk_direct_open},
2115        {sizeof(struct dk_cxlflash_release), (sioctl)cxlflash_disk_release},
2116        {sizeof(struct dk_cxlflash_detach), (sioctl)cxlflash_disk_detach},
2117        {sizeof(struct dk_cxlflash_verify), (sioctl)cxlflash_disk_verify},
2118        {sizeof(struct dk_cxlflash_recover_afu), (sioctl)cxlflash_afu_recover},
2119        {sizeof(struct dk_cxlflash_manage_lun), (sioctl)cxlflash_manage_lun},
2120        {sizeof(struct dk_cxlflash_uvirtual), cxlflash_disk_virtual_open},
2121        {sizeof(struct dk_cxlflash_resize), (sioctl)cxlflash_vlun_resize},
2122        {sizeof(struct dk_cxlflash_clone), (sioctl)cxlflash_disk_clone},
2123        };
2124
2125        /* Hold read semaphore so we can drain if needed */
2126        down_read(&cfg->ioctl_rwsem);
2127
2128        /* Restrict command set to physical support only for internal LUN */
2129        if (afu->internal_lun)
2130                switch (cmd) {
2131                case DK_CXLFLASH_RELEASE:
2132                case DK_CXLFLASH_USER_VIRTUAL:
2133                case DK_CXLFLASH_VLUN_RESIZE:
2134                case DK_CXLFLASH_VLUN_CLONE:
2135                        dev_dbg(dev, "%s: %s not supported for lun_mode=%d\n",
2136                                __func__, decode_ioctl(cmd), afu->internal_lun);
2137                        rc = -EINVAL;
2138                        goto cxlflash_ioctl_exit;
2139                }
2140
2141        switch (cmd) {
2142        case DK_CXLFLASH_ATTACH:
2143        case DK_CXLFLASH_USER_DIRECT:
2144        case DK_CXLFLASH_RELEASE:
2145        case DK_CXLFLASH_DETACH:
2146        case DK_CXLFLASH_VERIFY:
2147        case DK_CXLFLASH_RECOVER_AFU:
2148        case DK_CXLFLASH_USER_VIRTUAL:
2149        case DK_CXLFLASH_VLUN_RESIZE:
2150        case DK_CXLFLASH_VLUN_CLONE:
2151                dev_dbg(dev, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n",
2152                        __func__, decode_ioctl(cmd), cmd, shost->host_no,
2153                        sdev->channel, sdev->id, sdev->lun);
2154                rc = ioctl_common(sdev, cmd);
2155                if (unlikely(rc))
2156                        goto cxlflash_ioctl_exit;
2157
2158                fallthrough;
2159
2160        case DK_CXLFLASH_MANAGE_LUN:
2161                known_ioctl = true;
2162                idx = _IOC_NR(cmd) - _IOC_NR(DK_CXLFLASH_ATTACH);
2163                size = ioctl_tbl[idx].size;
2164                do_ioctl = ioctl_tbl[idx].ioctl;
2165
2166                if (likely(do_ioctl))
2167                        break;
2168
2169                fallthrough;
2170        default:
2171                rc = -EINVAL;
2172                goto cxlflash_ioctl_exit;
2173        }
2174
2175        if (unlikely(copy_from_user(&buf, arg, size))) {
2176                dev_err(dev, "%s: copy_from_user() fail size=%lu cmd=%u (%s) arg=%p\n",
2177                        __func__, size, cmd, decode_ioctl(cmd), arg);
2178                rc = -EFAULT;
2179                goto cxlflash_ioctl_exit;
2180        }
2181
2182        hdr = (struct dk_cxlflash_hdr *)&buf;
2183        if (hdr->version != DK_CXLFLASH_VERSION_0) {
2184                dev_dbg(dev, "%s: Version %u not supported for %s\n",
2185                        __func__, hdr->version, decode_ioctl(cmd));
2186                rc = -EINVAL;
2187                goto cxlflash_ioctl_exit;
2188        }
2189
2190        if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->rsvd[2] || hdr->return_flags) {
2191                dev_dbg(dev, "%s: Reserved/rflags populated\n", __func__);
2192                rc = -EINVAL;
2193                goto cxlflash_ioctl_exit;
2194        }
2195
2196        rc = do_ioctl(sdev, (void *)&buf);
2197        if (likely(!rc))
2198                if (unlikely(copy_to_user(arg, &buf, size))) {
2199                        dev_err(dev, "%s: copy_to_user() fail size=%lu cmd=%u (%s) arg=%p\n",
2200                                __func__, size, cmd, decode_ioctl(cmd), arg);
2201                        rc = -EFAULT;
2202                }
2203
2204        /* fall through to exit */
2205
2206cxlflash_ioctl_exit:
2207        up_read(&cfg->ioctl_rwsem);
2208        if (unlikely(rc && known_ioctl))
2209                dev_err(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2210                        "returned rc %d\n", __func__,
2211                        decode_ioctl(cmd), cmd, shost->host_no,
2212                        sdev->channel, sdev->id, sdev->lun, rc);
2213        else
2214                dev_dbg(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2215                        "returned rc %d\n", __func__, decode_ioctl(cmd),
2216                        cmd, shost->host_no, sdev->channel, sdev->id,
2217                        sdev->lun, rc);
2218        return rc;
2219}
2220