LXR linux/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2019, The Linux Foundation. All rights reserved.
   4 */
   5
   6#include <linux/acpi.h>
   7#include <linux/adreno-smmu-priv.h>
   8#include <linux/of_device.h>
   9#include <linux/qcom_scm.h>
  10
  11#include "arm-smmu.h"
  12
  13struct qcom_smmu {
  14        struct arm_smmu_device smmu;
  15        bool bypass_quirk;
  16        u8 bypass_cbndx;
  17        u32 stall_enabled;
  18};
  19
  20static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
  21{
  22        return container_of(smmu, struct qcom_smmu, smmu);
  23}
  24
  25static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx,
  26                u32 reg)
  27{
  28        struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
  29
  30        /*
  31         * On the GPU device we want to process subsequent transactions after a
  32         * fault to keep the GPU from hanging
  33         */
  34        reg |= ARM_SMMU_SCTLR_HUPCF;
  35
  36        if (qsmmu->stall_enabled & BIT(idx))
  37                reg |= ARM_SMMU_SCTLR_CFCFG;
  38
  39        arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
  40}
  41
  42static void qcom_adreno_smmu_get_fault_info(const void *cookie,
  43                struct adreno_smmu_fault_info *info)
  44{
  45        struct arm_smmu_domain *smmu_domain = (void *)cookie;
  46        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
  47        struct arm_smmu_device *smmu = smmu_domain->smmu;
  48
  49        info->fsr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSR);
  50        info->fsynr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR0);
  51        info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1);
  52        info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR);
  53        info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
  54        info->ttbr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
  55        info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR);
  56}
  57
  58static void qcom_adreno_smmu_set_stall(const void *cookie, bool enabled)
  59{
  60        struct arm_smmu_domain *smmu_domain = (void *)cookie;
  61        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
  62        struct qcom_smmu *qsmmu = to_qcom_smmu(smmu_domain->smmu);
  63
  64        if (enabled)
  65                qsmmu->stall_enabled |= BIT(cfg->cbndx);
  66        else
  67                qsmmu->stall_enabled &= ~BIT(cfg->cbndx);
  68}
  69
  70static void qcom_adreno_smmu_resume_translation(const void *cookie, bool terminate)
  71{
  72        struct arm_smmu_domain *smmu_domain = (void *)cookie;
  73        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
  74        struct arm_smmu_device *smmu = smmu_domain->smmu;
  75        u32 reg = 0;
  76
  77        if (terminate)
  78                reg |= ARM_SMMU_RESUME_TERMINATE;
  79
  80        arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_RESUME, reg);
  81}
  82
  83#define QCOM_ADRENO_SMMU_GPU_SID 0
  84
  85static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
  86{
  87        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  88        int i;
  89
  90        /*
  91         * The GPU will always use SID 0 so that is a handy way to uniquely
  92         * identify it and configure it for per-instance pagetables
  93         */
  94        for (i = 0; i < fwspec->num_ids; i++) {
  95                u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
  96
  97                if (sid == QCOM_ADRENO_SMMU_GPU_SID)
  98                        return true;
  99        }
 100
 101        return false;
 102}
 103
 104static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
 105                const void *cookie)
 106{
 107        struct arm_smmu_domain *smmu_domain = (void *)cookie;
 108        struct io_pgtable *pgtable =
 109                io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
 110        return &pgtable->cfg;
 111}
 112
 113/*
 114 * Local implementation to configure TTBR0 with the specified pagetable config.
 115 * The GPU driver will call this to enable TTBR0 when per-instance pagetables
 116 * are active
 117 */
 118
 119static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
 120                const struct io_pgtable_cfg *pgtbl_cfg)
 121{
 122        struct arm_smmu_domain *smmu_domain = (void *)cookie;
 123        struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
 124        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 125        struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 126
 127        /* The domain must have split pagetables already enabled */
 128        if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
 129                return -EINVAL;
 130
 131        /* If the pagetable config is NULL, disable TTBR0 */
 132        if (!pgtbl_cfg) {
 133                /* Do nothing if it is already disabled */
 134                if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
 135                        return -EINVAL;
 136
 137                /* Set TCR to the original configuration */
 138                cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
 139                cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
 140        } else {
 141                u32 tcr = cb->tcr[0];
 142
 143                /* Don't call this again if TTBR0 is already enabled */
 144                if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
 145                        return -EINVAL;
 146
 147                tcr |= arm_smmu_lpae_tcr(pgtbl_cfg);
 148                tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1);
 149
 150                cb->tcr[0] = tcr;
 151                cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 152                cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
 153        }
 154
 155        arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
 156
 157        return 0;
 158}
 159
 160static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
 161                                               struct arm_smmu_device *smmu,
 162                                               struct device *dev, int start)
 163{
 164        int count;
 165
 166        /*
 167         * Assign context bank 0 to the GPU device so the GPU hardware can
 168         * switch pagetables
 169         */
 170        if (qcom_adreno_smmu_is_gpu_device(dev)) {
 171                start = 0;
 172                count = 1;
 173        } else {
 174                start = 1;
 175                count = smmu->num_context_banks;
 176        }
 177
 178        return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
 179}
 180
 181static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
 182{
 183        const struct device_node *np = smmu->dev->of_node;
 184
 185        if (of_device_is_compatible(np, "qcom,msm8996-smmu-v2"))
 186                return false;
 187
 188        return true;
 189}
 190
 191static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
 192                struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
 193{
 194        struct adreno_smmu_priv *priv;
 195
 196        /* Only enable split pagetables for the GPU device (SID 0) */
 197        if (!qcom_adreno_smmu_is_gpu_device(dev))
 198                return 0;
 199
 200        /*
 201         * All targets that use the qcom,adreno-smmu compatible string *should*
 202         * be AARCH64 stage 1 but double check because the arm-smmu code assumes
 203         * that is the case when the TTBR1 quirk is enabled
 204         */
 205        if (qcom_adreno_can_do_ttbr1(smmu_domain->smmu) &&
 206            (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
 207            (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
 208                pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1;
 209
 210        /*
 211         * Initialize private interface with GPU:
 212         */
 213
 214        priv = dev_get_drvdata(dev);
 215        priv->cookie = smmu_domain;
 216        priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
 217        priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;
 218        priv->get_fault_info = qcom_adreno_smmu_get_fault_info;
 219        priv->set_stall = qcom_adreno_smmu_set_stall;
 220        priv->resume_translation = qcom_adreno_smmu_resume_translation;
 221
 222        return 0;
 223}
 224
 225static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
 226        { .compatible = "qcom,adreno" },
 227        { .compatible = "qcom,mdp4" },
 228        { .compatible = "qcom,mdss" },
 229        { .compatible = "qcom,sc7180-mdss" },
 230        { .compatible = "qcom,sc7180-mss-pil" },
 231        { .compatible = "qcom,sc7280-mdss" },
 232        { .compatible = "qcom,sc8180x-mdss" },
 233        { .compatible = "qcom,sdm845-mdss" },
 234        { .compatible = "qcom,sdm845-mss-pil" },
 235        { }
 236};
 237
 238static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
 239{
 240        unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
 241        struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
 242        u32 reg;
 243        u32 smr;
 244        int i;
 245
 246        /*
 247         * With some firmware versions writes to S2CR of type FAULT are
 248         * ignored, and writing BYPASS will end up written as FAULT in the
 249         * register. Perform a write to S2CR to detect if this is the case and
 250         * if so reserve a context bank to emulate bypass streams.
 251         */
 252        reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) |
 253              FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) |
 254              FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
 255        arm_smmu_gr0_write(smmu, last_s2cr, reg);
 256        reg = arm_smmu_gr0_read(smmu, last_s2cr);
 257        if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
 258                qsmmu->bypass_quirk = true;
 259                qsmmu->bypass_cbndx = smmu->num_context_banks - 1;
 260
 261                set_bit(qsmmu->bypass_cbndx, smmu->context_map);
 262
 263                arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);
 264
 265                reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
 266                arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
 267        }
 268
 269        for (i = 0; i < smmu->num_mapping_groups; i++) {
 270                smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
 271
 272                if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
 273                        /* Ignore valid bit for SMR mask extraction. */
 274                        smr &= ~ARM_SMMU_SMR_VALID;
 275                        smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
 276                        smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
 277                        smmu->smrs[i].valid = true;
 278
 279                        smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
 280                        smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
 281                        smmu->s2crs[i].cbndx = 0xff;
 282                }
 283        }
 284
 285        return 0;
 286}
 287
 288static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 289{
 290        struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 291        struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
 292        u32 cbndx = s2cr->cbndx;
 293        u32 type = s2cr->type;
 294        u32 reg;
 295
 296        if (qsmmu->bypass_quirk) {
 297                if (type == S2CR_TYPE_BYPASS) {
 298                        /*
 299                         * Firmware with quirky S2CR handling will substitute
 300                         * BYPASS writes with FAULT, so point the stream to the
 301                         * reserved context bank and ask for translation on the
 302                         * stream
 303                         */
 304                        type = S2CR_TYPE_TRANS;
 305                        cbndx = qsmmu->bypass_cbndx;
 306                } else if (type == S2CR_TYPE_FAULT) {
 307                        /*
 308                         * Firmware with quirky S2CR handling will ignore FAULT
 309                         * writes, so trick it to write FAULT by asking for a
 310                         * BYPASS.
 311                         */
 312                        type = S2CR_TYPE_BYPASS;
 313                        cbndx = 0xff;
 314                }
 315        }
 316
 317        reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) |
 318              FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) |
 319              FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
 320        arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 321}
 322
 323static int qcom_smmu_def_domain_type(struct device *dev)
 324{
 325        const struct of_device_id *match =
 326                of_match_device(qcom_smmu_client_of_match, dev);
 327
 328        return match ? IOMMU_DOMAIN_IDENTITY : 0;
 329}
 330
 331static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
 332{
 333        int ret;
 334
 335        /*
 336         * To address performance degradation in non-real time clients,
 337         * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
 338         * such as MTP and db845, whose firmwares implement secure monitor
 339         * call handlers to turn on/off the wait-for-safe logic.
 340         */
 341        ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
 342        if (ret)
 343                dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");
 344
 345        return ret;
 346}
 347
 348static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
 349{
 350        const struct device_node *np = smmu->dev->of_node;
 351
 352        arm_mmu500_reset(smmu);
 353
 354        if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
 355                return qcom_sdm845_smmu500_reset(smmu);
 356
 357        return 0;
 358}
 359
 360static const struct arm_smmu_impl qcom_smmu_impl = {
 361        .cfg_probe = qcom_smmu_cfg_probe,
 362        .def_domain_type = qcom_smmu_def_domain_type,
 363        .reset = qcom_smmu500_reset,
 364        .write_s2cr = qcom_smmu_write_s2cr,
 365};
 366
 367static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
 368        .init_context = qcom_adreno_smmu_init_context,
 369        .def_domain_type = qcom_smmu_def_domain_type,
 370        .reset = qcom_smmu500_reset,
 371        .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
 372        .write_sctlr = qcom_adreno_smmu_write_sctlr,
 373};
 374
 375static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
 376                const struct arm_smmu_impl *impl)
 377{
 378        struct qcom_smmu *qsmmu;
 379
 380        /* Check to make sure qcom_scm has finished probing */
 381        if (!qcom_scm_is_available())
 382                return ERR_PTR(-EPROBE_DEFER);
 383
 384        qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL);
 385        if (!qsmmu)
 386                return ERR_PTR(-ENOMEM);
 387
 388        qsmmu->smmu.impl = impl;
 389
 390        return &qsmmu->smmu;
 391}
 392
 393static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
 394        { .compatible = "qcom,msm8998-smmu-v2" },
 395        { .compatible = "qcom,sc7180-smmu-500" },
 396        { .compatible = "qcom,sc7280-smmu-500" },
 397        { .compatible = "qcom,sc8180x-smmu-500" },
 398        { .compatible = "qcom,sdm630-smmu-v2" },
 399        { .compatible = "qcom,sdm845-smmu-500" },
 400        { .compatible = "qcom,sm6125-smmu-500" },
 401        { .compatible = "qcom,sm8150-smmu-500" },
 402        { .compatible = "qcom,sm8250-smmu-500" },
 403        { .compatible = "qcom,sm8350-smmu-500" },
 404        { }
 405};
 406
 407#ifdef CONFIG_ACPI
 408static struct acpi_platform_list qcom_acpi_platlist[] = {
 409        { "LENOVO", "CB-01   ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
 410        { "QCOM  ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
 411        { }
 412};
 413#endif
 414
 415struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
 416{
 417        const struct device_node *np = smmu->dev->of_node;
 418
 419#ifdef CONFIG_ACPI
 420        if (np == NULL) {
 421                /* Match platform for ACPI boot */
 422                if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
 423                        return qcom_smmu_create(smmu, &qcom_smmu_impl);
 424        }
 425#endif
 426
 427        /*
 428         * Do not change this order of implementation, i.e., first adreno
 429         * smmu impl and then apss smmu since we can have both implementing
 430         * arm,mmu-500 in which case we will miss setting adreno smmu specific
 431         * features if the order is changed.
 432         */
 433        if (of_device_is_compatible(np, "qcom,adreno-smmu"))
 434                return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
 435
 436        if (of_match_node(qcom_smmu_impl_of_match, np))
 437                return qcom_smmu_create(smmu, &qcom_smmu_impl);
 438
 439        return smmu;
 440}
 441