linux/drivers/iommu/tegra-gart.c
<<
>>
Prefs
   1/*
   2 * IOMMU API for GART in Tegra20
   3 *
   4 * Copyright (c) 2010-2012, NVIDIA CORPORATION.  All rights reserved.
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms and conditions of the GNU General Public License,
   8 * version 2, as published by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope it will be useful, but WITHOUT
  11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 * more details.
  14 *
  15 * You should have received a copy of the GNU General Public License along with
  16 * this program; if not, write to the Free Software Foundation, Inc.,
  17 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20#define pr_fmt(fmt)     "%s(): " fmt, __func__
  21
  22#include <linux/module.h>
  23#include <linux/platform_device.h>
  24#include <linux/spinlock.h>
  25#include <linux/slab.h>
  26#include <linux/vmalloc.h>
  27#include <linux/mm.h>
  28#include <linux/list.h>
  29#include <linux/device.h>
  30#include <linux/io.h>
  31#include <linux/iommu.h>
  32#include <linux/of.h>
  33
  34#include <asm/cacheflush.h>
  35
  36/* bitmap of the page sizes currently supported */
  37#define GART_IOMMU_PGSIZES      (SZ_4K)
  38
  39#define GART_REG_BASE           0x24
  40#define GART_CONFIG             (0x24 - GART_REG_BASE)
  41#define GART_ENTRY_ADDR         (0x28 - GART_REG_BASE)
  42#define GART_ENTRY_DATA         (0x2c - GART_REG_BASE)
  43#define GART_ENTRY_PHYS_ADDR_VALID      (1 << 31)
  44
  45#define GART_PAGE_SHIFT         12
  46#define GART_PAGE_SIZE          (1 << GART_PAGE_SHIFT)
  47#define GART_PAGE_MASK                                          \
  48        (~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)
  49
  50struct gart_client {
  51        struct device           *dev;
  52        struct list_head        list;
  53};
  54
  55struct gart_device {
  56        void __iomem            *regs;
  57        u32                     *savedata;
  58        u32                     page_count;     /* total remappable size */
  59        dma_addr_t              iovmm_base;     /* offset to vmm_area */
  60        spinlock_t              pte_lock;       /* for pagetable */
  61        struct list_head        client;
  62        spinlock_t              client_lock;    /* for client list */
  63        struct device           *dev;
  64};
  65
  66static struct gart_device *gart_handle; /* unique for a system */
  67
  68#define GART_PTE(_pfn)                                          \
  69        (GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT))
  70
  71/*
  72 * Any interaction between any block on PPSB and a block on APB or AHB
  73 * must have these read-back to ensure the APB/AHB bus transaction is
  74 * complete before initiating activity on the PPSB block.
  75 */
  76#define FLUSH_GART_REGS(gart)   ((void)readl((gart)->regs + GART_CONFIG))
  77
  78#define for_each_gart_pte(gart, iova)                                   \
  79        for (iova = gart->iovmm_base;                                   \
  80             iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
  81             iova += GART_PAGE_SIZE)
  82
  83static inline void gart_set_pte(struct gart_device *gart,
  84                                unsigned long offs, u32 pte)
  85{
  86        writel(offs, gart->regs + GART_ENTRY_ADDR);
  87        writel(pte, gart->regs + GART_ENTRY_DATA);
  88
  89        dev_dbg(gart->dev, "%s %08lx:%08x\n",
  90                 pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
  91}
  92
  93static inline unsigned long gart_read_pte(struct gart_device *gart,
  94                                          unsigned long offs)
  95{
  96        unsigned long pte;
  97
  98        writel(offs, gart->regs + GART_ENTRY_ADDR);
  99        pte = readl(gart->regs + GART_ENTRY_DATA);
 100
 101        return pte;
 102}
 103
 104static void do_gart_setup(struct gart_device *gart, const u32 *data)
 105{
 106        unsigned long iova;
 107
 108        for_each_gart_pte(gart, iova)
 109                gart_set_pte(gart, iova, data ? *(data++) : 0);
 110
 111        writel(1, gart->regs + GART_CONFIG);
 112        FLUSH_GART_REGS(gart);
 113}
 114
 115#ifdef DEBUG
 116static void gart_dump_table(struct gart_device *gart)
 117{
 118        unsigned long iova;
 119        unsigned long flags;
 120
 121        spin_lock_irqsave(&gart->pte_lock, flags);
 122        for_each_gart_pte(gart, iova) {
 123                unsigned long pte;
 124
 125                pte = gart_read_pte(gart, iova);
 126
 127                dev_dbg(gart->dev, "%s %08lx:%08lx\n",
 128                        (GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
 129                        iova, pte & GART_PAGE_MASK);
 130        }
 131        spin_unlock_irqrestore(&gart->pte_lock, flags);
 132}
 133#else
 134static inline void gart_dump_table(struct gart_device *gart)
 135{
 136}
 137#endif
 138
 139static inline bool gart_iova_range_valid(struct gart_device *gart,
 140                                         unsigned long iova, size_t bytes)
 141{
 142        unsigned long iova_start, iova_end, gart_start, gart_end;
 143
 144        iova_start = iova;
 145        iova_end = iova_start + bytes - 1;
 146        gart_start = gart->iovmm_base;
 147        gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;
 148
 149        if (iova_start < gart_start)
 150                return false;
 151        if (iova_end > gart_end)
 152                return false;
 153        return true;
 154}
 155
 156static int gart_iommu_attach_dev(struct iommu_domain *domain,
 157                                 struct device *dev)
 158{
 159        struct gart_device *gart;
 160        struct gart_client *client, *c;
 161        int err = 0;
 162
 163        gart = gart_handle;
 164        if (!gart)
 165                return -EINVAL;
 166        domain->priv = gart;
 167
 168        domain->geometry.aperture_start = gart->iovmm_base;
 169        domain->geometry.aperture_end   = gart->iovmm_base +
 170                                        gart->page_count * GART_PAGE_SIZE - 1;
 171        domain->geometry.force_aperture = true;
 172
 173        client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);
 174        if (!client)
 175                return -ENOMEM;
 176        client->dev = dev;
 177
 178        spin_lock(&gart->client_lock);
 179        list_for_each_entry(c, &gart->client, list) {
 180                if (c->dev == dev) {
 181                        dev_err(gart->dev,
 182                                "%s is already attached\n", dev_name(dev));
 183                        err = -EINVAL;
 184                        goto fail;
 185                }
 186        }
 187        list_add(&client->list, &gart->client);
 188        spin_unlock(&gart->client_lock);
 189        dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
 190        return 0;
 191
 192fail:
 193        devm_kfree(gart->dev, client);
 194        spin_unlock(&gart->client_lock);
 195        return err;
 196}
 197
 198static void gart_iommu_detach_dev(struct iommu_domain *domain,
 199                                  struct device *dev)
 200{
 201        struct gart_device *gart = domain->priv;
 202        struct gart_client *c;
 203
 204        spin_lock(&gart->client_lock);
 205
 206        list_for_each_entry(c, &gart->client, list) {
 207                if (c->dev == dev) {
 208                        list_del(&c->list);
 209                        devm_kfree(gart->dev, c);
 210                        dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
 211                        goto out;
 212                }
 213        }
 214        dev_err(gart->dev, "Couldn't find\n");
 215out:
 216        spin_unlock(&gart->client_lock);
 217}
 218
 219static int gart_iommu_domain_init(struct iommu_domain *domain)
 220{
 221        return 0;
 222}
 223
 224static void gart_iommu_domain_destroy(struct iommu_domain *domain)
 225{
 226        struct gart_device *gart = domain->priv;
 227
 228        if (!gart)
 229                return;
 230
 231        spin_lock(&gart->client_lock);
 232        if (!list_empty(&gart->client)) {
 233                struct gart_client *c;
 234
 235                list_for_each_entry(c, &gart->client, list)
 236                        gart_iommu_detach_dev(domain, c->dev);
 237        }
 238        spin_unlock(&gart->client_lock);
 239        domain->priv = NULL;
 240}
 241
 242static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
 243                          phys_addr_t pa, size_t bytes, int prot)
 244{
 245        struct gart_device *gart = domain->priv;
 246        unsigned long flags;
 247        unsigned long pfn;
 248
 249        if (!gart_iova_range_valid(gart, iova, bytes))
 250                return -EINVAL;
 251
 252        spin_lock_irqsave(&gart->pte_lock, flags);
 253        pfn = __phys_to_pfn(pa);
 254        if (!pfn_valid(pfn)) {
 255                dev_err(gart->dev, "Invalid page: %08x\n", pa);
 256                spin_unlock_irqrestore(&gart->pte_lock, flags);
 257                return -EINVAL;
 258        }
 259        gart_set_pte(gart, iova, GART_PTE(pfn));
 260        FLUSH_GART_REGS(gart);
 261        spin_unlock_irqrestore(&gart->pte_lock, flags);
 262        return 0;
 263}
 264
 265static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 266                               size_t bytes)
 267{
 268        struct gart_device *gart = domain->priv;
 269        unsigned long flags;
 270
 271        if (!gart_iova_range_valid(gart, iova, bytes))
 272                return 0;
 273
 274        spin_lock_irqsave(&gart->pte_lock, flags);
 275        gart_set_pte(gart, iova, 0);
 276        FLUSH_GART_REGS(gart);
 277        spin_unlock_irqrestore(&gart->pte_lock, flags);
 278        return 0;
 279}
 280
 281static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
 282                                           unsigned long iova)
 283{
 284        struct gart_device *gart = domain->priv;
 285        unsigned long pte;
 286        phys_addr_t pa;
 287        unsigned long flags;
 288
 289        if (!gart_iova_range_valid(gart, iova, 0))
 290                return -EINVAL;
 291
 292        spin_lock_irqsave(&gart->pte_lock, flags);
 293        pte = gart_read_pte(gart, iova);
 294        spin_unlock_irqrestore(&gart->pte_lock, flags);
 295
 296        pa = (pte & GART_PAGE_MASK);
 297        if (!pfn_valid(__phys_to_pfn(pa))) {
 298                dev_err(gart->dev, "No entry for %08lx:%08x\n", iova, pa);
 299                gart_dump_table(gart);
 300                return -EINVAL;
 301        }
 302        return pa;
 303}
 304
 305static int gart_iommu_domain_has_cap(struct iommu_domain *domain,
 306                                     unsigned long cap)
 307{
 308        return 0;
 309}
 310
 311static struct iommu_ops gart_iommu_ops = {
 312        .domain_init    = gart_iommu_domain_init,
 313        .domain_destroy = gart_iommu_domain_destroy,
 314        .attach_dev     = gart_iommu_attach_dev,
 315        .detach_dev     = gart_iommu_detach_dev,
 316        .map            = gart_iommu_map,
 317        .unmap          = gart_iommu_unmap,
 318        .iova_to_phys   = gart_iommu_iova_to_phys,
 319        .domain_has_cap = gart_iommu_domain_has_cap,
 320        .pgsize_bitmap  = GART_IOMMU_PGSIZES,
 321};
 322
 323static int tegra_gart_suspend(struct device *dev)
 324{
 325        struct gart_device *gart = dev_get_drvdata(dev);
 326        unsigned long iova;
 327        u32 *data = gart->savedata;
 328        unsigned long flags;
 329
 330        spin_lock_irqsave(&gart->pte_lock, flags);
 331        for_each_gart_pte(gart, iova)
 332                *(data++) = gart_read_pte(gart, iova);
 333        spin_unlock_irqrestore(&gart->pte_lock, flags);
 334        return 0;
 335}
 336
 337static int tegra_gart_resume(struct device *dev)
 338{
 339        struct gart_device *gart = dev_get_drvdata(dev);
 340        unsigned long flags;
 341
 342        spin_lock_irqsave(&gart->pte_lock, flags);
 343        do_gart_setup(gart, gart->savedata);
 344        spin_unlock_irqrestore(&gart->pte_lock, flags);
 345        return 0;
 346}
 347
 348static int tegra_gart_probe(struct platform_device *pdev)
 349{
 350        struct gart_device *gart;
 351        struct resource *res, *res_remap;
 352        void __iomem *gart_regs;
 353        int err;
 354        struct device *dev = &pdev->dev;
 355
 356        if (gart_handle)
 357                return -EIO;
 358
 359        BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
 360
 361        /* the GART memory aperture is required */
 362        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 363        res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 364        if (!res || !res_remap) {
 365                dev_err(dev, "GART memory aperture expected\n");
 366                return -ENXIO;
 367        }
 368
 369        gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL);
 370        if (!gart) {
 371                dev_err(dev, "failed to allocate gart_device\n");
 372                return -ENOMEM;
 373        }
 374
 375        gart_regs = devm_ioremap(dev, res->start, resource_size(res));
 376        if (!gart_regs) {
 377                dev_err(dev, "failed to remap GART registers\n");
 378                err = -ENXIO;
 379                goto fail;
 380        }
 381
 382        gart->dev = &pdev->dev;
 383        spin_lock_init(&gart->pte_lock);
 384        spin_lock_init(&gart->client_lock);
 385        INIT_LIST_HEAD(&gart->client);
 386        gart->regs = gart_regs;
 387        gart->iovmm_base = (dma_addr_t)res_remap->start;
 388        gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);
 389
 390        gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
 391        if (!gart->savedata) {
 392                dev_err(dev, "failed to allocate context save area\n");
 393                err = -ENOMEM;
 394                goto fail;
 395        }
 396
 397        platform_set_drvdata(pdev, gart);
 398        do_gart_setup(gart, NULL);
 399
 400        gart_handle = gart;
 401        bus_set_iommu(&platform_bus_type, &gart_iommu_ops);
 402        return 0;
 403
 404fail:
 405        if (gart_regs)
 406                devm_iounmap(dev, gart_regs);
 407        if (gart && gart->savedata)
 408                vfree(gart->savedata);
 409        devm_kfree(dev, gart);
 410        return err;
 411}
 412
 413static int tegra_gart_remove(struct platform_device *pdev)
 414{
 415        struct gart_device *gart = platform_get_drvdata(pdev);
 416        struct device *dev = gart->dev;
 417
 418        writel(0, gart->regs + GART_CONFIG);
 419        if (gart->savedata)
 420                vfree(gart->savedata);
 421        if (gart->regs)
 422                devm_iounmap(dev, gart->regs);
 423        devm_kfree(dev, gart);
 424        gart_handle = NULL;
 425        return 0;
 426}
 427
 428const struct dev_pm_ops tegra_gart_pm_ops = {
 429        .suspend        = tegra_gart_suspend,
 430        .resume         = tegra_gart_resume,
 431};
 432
 433#ifdef CONFIG_OF
 434static struct of_device_id tegra_gart_of_match[] = {
 435        { .compatible = "nvidia,tegra20-gart", },
 436        { },
 437};
 438MODULE_DEVICE_TABLE(of, tegra_gart_of_match);
 439#endif
 440
 441static struct platform_driver tegra_gart_driver = {
 442        .probe          = tegra_gart_probe,
 443        .remove         = tegra_gart_remove,
 444        .driver = {
 445                .owner  = THIS_MODULE,
 446                .name   = "tegra-gart",
 447                .pm     = &tegra_gart_pm_ops,
 448                .of_match_table = of_match_ptr(tegra_gart_of_match),
 449        },
 450};
 451
 452static int tegra_gart_init(void)
 453{
 454        return platform_driver_register(&tegra_gart_driver);
 455}
 456
 457static void __exit tegra_gart_exit(void)
 458{
 459        platform_driver_unregister(&tegra_gart_driver);
 460}
 461
 462subsys_initcall(tegra_gart_init);
 463module_exit(tegra_gart_exit);
 464
 465MODULE_DESCRIPTION("IOMMU API for GART in Tegra20");
 466MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
 467MODULE_ALIAS("platform:tegra-gart");
 468MODULE_LICENSE("GPL v2");
 469
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.